1
2#ifndef _INTEL_RINGBUFFER_H_
3#define _INTEL_RINGBUFFER_H_
4
5#include <linux/hashtable.h>
6#include "i915_gem_batch_pool.h"
7#include "i915_gem_request.h"
8#include "i915_gem_timeline.h"
9#include "i915_selftest.h"
10
11#define I915_CMD_HASH_ORDER 9
12
13
14
15
16
17
18#define CACHELINE_BYTES 64
19#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
20
21struct intel_hw_status_page {
22 struct i915_vma *vma;
23 u32 *page_addr;
24 u32 ggtt_offset;
25};
26
27#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
28#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
29
30#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
31#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
32
33#define I915_READ_HEAD(engine) I915_READ(RING_HEAD((engine)->mmio_base))
34#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
35
36#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
37#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
38
39#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
40#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
41
42#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
43#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
44
45
46
47
48#define gen8_semaphore_seqno_size sizeof(uint64_t)
49#define GEN8_SEMAPHORE_OFFSET(__from, __to) \
50 (((__from) * I915_NUM_ENGINES + (__to)) * gen8_semaphore_seqno_size)
51#define GEN8_SIGNAL_OFFSET(__ring, to) \
52 (dev_priv->semaphore->node.start + \
53 GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
54#define GEN8_WAIT_OFFSET(__ring, from) \
55 (dev_priv->semaphore->node.start + \
56 GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
57
58enum intel_engine_hangcheck_action {
59 ENGINE_IDLE = 0,
60 ENGINE_WAIT,
61 ENGINE_ACTIVE_SEQNO,
62 ENGINE_ACTIVE_HEAD,
63 ENGINE_ACTIVE_SUBUNITS,
64 ENGINE_WAIT_KICK,
65 ENGINE_DEAD,
66};
67
68static inline const char *
69hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
70{
71 switch (a) {
72 case ENGINE_IDLE:
73 return "idle";
74 case ENGINE_WAIT:
75 return "wait";
76 case ENGINE_ACTIVE_SEQNO:
77 return "active seqno";
78 case ENGINE_ACTIVE_HEAD:
79 return "active head";
80 case ENGINE_ACTIVE_SUBUNITS:
81 return "active subunits";
82 case ENGINE_WAIT_KICK:
83 return "wait kick";
84 case ENGINE_DEAD:
85 return "dead";
86 }
87
88 return "unknown";
89}
90
91#define I915_MAX_SLICES 3
92#define I915_MAX_SUBSLICES 3
93
94#define instdone_slice_mask(dev_priv__) \
95 (INTEL_GEN(dev_priv__) == 7 ? \
96 1 : INTEL_INFO(dev_priv__)->sseu.slice_mask)
97
98#define instdone_subslice_mask(dev_priv__) \
99 (INTEL_GEN(dev_priv__) == 7 ? \
100 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask)
101
102#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
103 for ((slice__) = 0, (subslice__) = 0; \
104 (slice__) < I915_MAX_SLICES; \
105 (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
106 (slice__) += ((subslice__) == 0)) \
107 for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
108 (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
109
110struct intel_instdone {
111 u32 instdone;
112
113 u32 slice_common;
114 u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
115 u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
116};
117
118struct intel_engine_hangcheck {
119 u64 acthd;
120 u32 seqno;
121 enum intel_engine_hangcheck_action action;
122 unsigned long action_timestamp;
123 int deadlock;
124 struct intel_instdone instdone;
125 struct drm_i915_gem_request *active_request;
126 bool stalled;
127};
128
129struct intel_ring {
130 struct i915_vma *vma;
131 void *vaddr;
132
133 struct list_head request_list;
134
135 u32 head;
136 u32 tail;
137 u32 emit;
138
139 u32 space;
140 u32 size;
141 u32 effective_size;
142};
143
144struct i915_gem_context;
145struct drm_i915_reg_table;
146
147
148
149
150
151
152
153
154
155
156
157
158struct i915_ctx_workarounds {
159 struct i915_wa_ctx_bb {
160 u32 offset;
161 u32 size;
162 } indirect_ctx, per_ctx;
163 struct i915_vma *vma;
164};
165
166struct drm_i915_gem_request;
167struct intel_render_state;
168
169
170
171
172
173enum intel_engine_id {
174 RCS = 0,
175 BCS,
176 VCS,
177 VCS2,
178#define _VCS(n) (VCS + (n))
179 VECS
180};
181
182struct i915_priolist {
183 struct rb_node node;
184 struct list_head requests;
185 int priority;
186};
187
188#define INTEL_ENGINE_CS_MAX_NAME 8
189
190struct intel_engine_cs {
191 struct drm_i915_private *i915;
192 char name[INTEL_ENGINE_CS_MAX_NAME];
193 enum intel_engine_id id;
194 unsigned int uabi_id;
195 unsigned int hw_id;
196 unsigned int guc_id;
197
198 u8 class;
199 u8 instance;
200 u32 context_size;
201 u32 mmio_base;
202 unsigned int irq_shift;
203
204 struct intel_ring *buffer;
205 struct intel_timeline *timeline;
206
207 struct intel_render_state *render_state;
208
209 atomic_t irq_count;
210 unsigned long irq_posted;
211#define ENGINE_IRQ_BREADCRUMB 0
212#define ENGINE_IRQ_EXECLIST 1
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230 struct intel_breadcrumbs {
231 spinlock_t irq_lock;
232 struct intel_wait *irq_wait;
233
234 spinlock_t rb_lock;
235 struct rb_root waiters;
236 struct rb_root signals;
237 struct task_struct *signaler;
238 struct drm_i915_gem_request __rcu *first_signal;
239 struct timer_list fake_irq;
240 struct timer_list hangcheck;
241
242 unsigned int hangcheck_interrupts;
243
244 bool irq_armed : 1;
245 bool irq_enabled : 1;
246 I915_SELFTEST_DECLARE(bool mock : 1);
247 } breadcrumbs;
248
249
250
251
252
253
254 struct i915_gem_batch_pool batch_pool;
255
256 struct intel_hw_status_page status_page;
257 struct i915_ctx_workarounds wa_ctx;
258 struct i915_vma *scratch;
259
260 u32 irq_keep_mask;
261 u32 irq_enable_mask;
262 void (*irq_enable)(struct intel_engine_cs *engine);
263 void (*irq_disable)(struct intel_engine_cs *engine);
264
265 int (*init_hw)(struct intel_engine_cs *engine);
266 void (*reset_hw)(struct intel_engine_cs *engine,
267 struct drm_i915_gem_request *req);
268
269 void (*set_default_submission)(struct intel_engine_cs *engine);
270
271 struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
272 struct i915_gem_context *ctx);
273 void (*context_unpin)(struct intel_engine_cs *engine,
274 struct i915_gem_context *ctx);
275 int (*request_alloc)(struct drm_i915_gem_request *req);
276 int (*init_context)(struct drm_i915_gem_request *req);
277
278 int (*emit_flush)(struct drm_i915_gem_request *request,
279 u32 mode);
280#define EMIT_INVALIDATE BIT(0)
281#define EMIT_FLUSH BIT(1)
282#define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH)
283 int (*emit_bb_start)(struct drm_i915_gem_request *req,
284 u64 offset, u32 length,
285 unsigned int dispatch_flags);
286#define I915_DISPATCH_SECURE BIT(0)
287#define I915_DISPATCH_PINNED BIT(1)
288#define I915_DISPATCH_RS BIT(2)
289 void (*emit_breadcrumb)(struct drm_i915_gem_request *req,
290 u32 *cs);
291 int emit_breadcrumb_sz;
292
293
294
295
296
297
298
299 void (*submit_request)(struct drm_i915_gem_request *req);
300
301
302
303
304
305
306
307 void (*schedule)(struct drm_i915_gem_request *request,
308 int priority);
309
310
311
312
313
314
315
316 void (*irq_seqno_barrier)(struct intel_engine_cs *engine);
317 void (*cleanup)(struct intel_engine_cs *engine);
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356 struct {
357 union {
358#define GEN6_SEMAPHORE_LAST VECS_HW
359#define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1)
360#define GEN6_SEMAPHORES_MASK GENMASK(GEN6_SEMAPHORE_LAST, 0)
361 struct {
362
363 u32 wait[GEN6_NUM_SEMAPHORES];
364
365 i915_reg_t signal[GEN6_NUM_SEMAPHORES];
366 } mbox;
367 u64 signal_ggtt[I915_NUM_ENGINES];
368 };
369
370
371 int (*sync_to)(struct drm_i915_gem_request *req,
372 struct drm_i915_gem_request *signal);
373 u32 *(*signal)(struct drm_i915_gem_request *req, u32 *cs);
374 } semaphore;
375
376
377 struct tasklet_struct irq_tasklet;
378 struct i915_priolist default_priolist;
379 bool no_priolist;
380 struct execlist_port {
381 struct drm_i915_gem_request *request_count;
382#define EXECLIST_COUNT_BITS 2
383#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
384#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
385#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
386#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
387#define port_set(p, packed) ((p)->request_count = (packed))
388#define port_isset(p) ((p)->request_count)
389#define port_index(p, e) ((p) - (e)->execlist_port)
390 GEM_DEBUG_DECL(u32 context_id);
391 } execlist_port[2];
392 struct rb_root execlist_queue;
393 struct rb_node *execlist_first;
394 unsigned int fw_domains;
395
396
397
398
399
400
401
402
403
404
405 struct i915_gem_context *last_retired_context;
406
407
408
409
410
411
412
413 struct i915_gem_context *legacy_active_context;
414
415
416 struct atomic_notifier_head context_status_notifier;
417
418 struct intel_engine_hangcheck hangcheck;
419
420 bool needs_cmd_parser;
421
422
423
424
425
426 DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
427
428
429
430
431 const struct drm_i915_reg_table *reg_tables;
432 int reg_table_count;
433
434
435
436
437
438
439
440
441
442
443
444 u32 (*get_cmd_length_mask)(u32 cmd_header);
445};
446
447static inline unsigned int
448intel_engine_flag(const struct intel_engine_cs *engine)
449{
450 return BIT(engine->id);
451}
452
453static inline u32
454intel_read_status_page(struct intel_engine_cs *engine, int reg)
455{
456
457 return READ_ONCE(engine->status_page.page_addr[reg]);
458}
459
460static inline void
461intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
462{
463
464
465
466
467
468 if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
469 mb();
470 clflush(&engine->status_page.page_addr[reg]);
471 engine->status_page.page_addr[reg] = value;
472 clflush(&engine->status_page.page_addr[reg]);
473 mb();
474 } else {
475 WRITE_ONCE(engine->status_page.page_addr[reg], value);
476 }
477}
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495#define I915_GEM_HWS_INDEX 0x30
496#define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
497#define I915_GEM_HWS_SCRATCH_INDEX 0x40
498#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
499
500struct intel_ring *
501intel_engine_create_ring(struct intel_engine_cs *engine, int size);
502int intel_ring_pin(struct intel_ring *ring,
503 struct drm_i915_private *i915,
504 unsigned int offset_bias);
505void intel_ring_reset(struct intel_ring *ring, u32 tail);
506unsigned int intel_ring_update_space(struct intel_ring *ring);
507void intel_ring_unpin(struct intel_ring *ring);
508void intel_ring_free(struct intel_ring *ring);
509
510void intel_engine_stop(struct intel_engine_cs *engine);
511void intel_engine_cleanup(struct intel_engine_cs *engine);
512
513void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
514
515int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
516
517u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
518 unsigned int n);
519
520static inline void
521intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
522{
523
524
525
526
527
528
529
530
531 GEM_BUG_ON((req->ring->vaddr + req->ring->emit) != cs);
532}
533
534static inline u32
535intel_ring_wrap(const struct intel_ring *ring, u32 pos)
536{
537 return pos & (ring->size - 1);
538}
539
540static inline u32
541intel_ring_offset(const struct drm_i915_gem_request *req, void *addr)
542{
543
544 u32 offset = addr - req->ring->vaddr;
545 GEM_BUG_ON(offset > req->ring->size);
546 return intel_ring_wrap(req->ring, offset);
547}
548
549static inline void
550assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
551{
552
553
554
555
556 GEM_BUG_ON(!IS_ALIGNED(tail, 8));
557 GEM_BUG_ON(tail >= ring->size);
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573#define cacheline(a) round_down(a, CACHELINE_BYTES)
574 GEM_BUG_ON(cacheline(tail) == cacheline(ring->head) &&
575 tail < ring->head);
576#undef cacheline
577}
578
579static inline unsigned int
580intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
581{
582
583
584
585
586
587
588 assert_ring_tail_valid(ring, tail);
589 ring->tail = tail;
590 return tail;
591}
592
593void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno);
594
595void intel_engine_setup_common(struct intel_engine_cs *engine);
596int intel_engine_init_common(struct intel_engine_cs *engine);
597int intel_engine_create_scratch(struct intel_engine_cs *engine, int size);
598void intel_engine_cleanup_common(struct intel_engine_cs *engine);
599
600int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
601int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
602int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
603int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
604
605u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
606u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine);
607
608static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
609{
610 return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
611}
612
613static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
614{
615
616
617
618
619
620
621
622 return READ_ONCE(engine->timeline->seqno);
623}
624
625int init_workarounds_ring(struct intel_engine_cs *engine);
626int intel_ring_workarounds_emit(struct drm_i915_gem_request *req);
627
628void intel_engine_get_instdone(struct intel_engine_cs *engine,
629 struct intel_instdone *instdone);
630
631
632
633
634
635
636
637
638#define MIN_SPACE_FOR_ADD_REQUEST 336
639
640static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
641{
642 return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
643}
644
645
646int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
647
648static inline void intel_wait_init(struct intel_wait *wait,
649 struct drm_i915_gem_request *rq)
650{
651 wait->tsk = current;
652 wait->request = rq;
653}
654
655static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno)
656{
657 wait->tsk = current;
658 wait->seqno = seqno;
659}
660
661static inline bool intel_wait_has_seqno(const struct intel_wait *wait)
662{
663 return wait->seqno;
664}
665
666static inline bool
667intel_wait_update_seqno(struct intel_wait *wait, u32 seqno)
668{
669 wait->seqno = seqno;
670 return intel_wait_has_seqno(wait);
671}
672
673static inline bool
674intel_wait_update_request(struct intel_wait *wait,
675 const struct drm_i915_gem_request *rq)
676{
677 return intel_wait_update_seqno(wait, i915_gem_request_global_seqno(rq));
678}
679
680static inline bool
681intel_wait_check_seqno(const struct intel_wait *wait, u32 seqno)
682{
683 return wait->seqno == seqno;
684}
685
686static inline bool
687intel_wait_check_request(const struct intel_wait *wait,
688 const struct drm_i915_gem_request *rq)
689{
690 return intel_wait_check_seqno(wait, i915_gem_request_global_seqno(rq));
691}
692
693static inline bool intel_wait_complete(const struct intel_wait *wait)
694{
695 return RB_EMPTY_NODE(&wait->node);
696}
697
698bool intel_engine_add_wait(struct intel_engine_cs *engine,
699 struct intel_wait *wait);
700void intel_engine_remove_wait(struct intel_engine_cs *engine,
701 struct intel_wait *wait);
702void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
703 bool wakeup);
704void intel_engine_cancel_signaling(struct drm_i915_gem_request *request);
705
706static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
707{
708 return READ_ONCE(engine->breadcrumbs.irq_wait);
709}
710
711unsigned int intel_engine_wakeup(struct intel_engine_cs *engine);
712#define ENGINE_WAKEUP_WAITER BIT(0)
713#define ENGINE_WAKEUP_ASLEEP BIT(1)
714
715void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
716void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
717
718void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
719void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
720bool intel_breadcrumbs_busy(struct intel_engine_cs *engine);
721
722static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
723{
724 memset(batch, 0, 6 * sizeof(u32));
725
726 batch[0] = GFX_OP_PIPE_CONTROL(6);
727 batch[1] = flags;
728 batch[2] = offset;
729
730 return batch + 6;
731}
732
733bool intel_engine_is_idle(struct intel_engine_cs *engine);
734bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
735
736void intel_engines_mark_idle(struct drm_i915_private *i915);
737void intel_engines_reset_default_submission(struct drm_i915_private *i915);
738
739static inline bool
740__intel_engine_can_store_dword(unsigned int gen, unsigned int class)
741{
742 if (gen <= 2)
743 return false;
744
745 if (gen == 6 && class == VIDEO_DECODE_CLASS)
746 return false;
747
748 return true;
749}
750
751#endif
752