1
2#ifndef _INTEL_RINGBUFFER_H_
3#define _INTEL_RINGBUFFER_H_
4
5#include <drm/drm_util.h>
6
7#include <linux/hashtable.h>
8#include <linux/irq_work.h>
9#include <linux/random.h>
10#include <linux/seqlock.h>
11
12#include "i915_pmu.h"
13#include "i915_reg.h"
14#include "i915_request.h"
15#include "i915_selftest.h"
16#include "intel_engine_types.h"
17#include "intel_gt_types.h"
18#include "intel_timeline.h"
19#include "intel_workarounds.h"
20
21struct drm_printer;
22struct intel_context;
23struct intel_gt;
24struct lock_class_key;
25
26
27
28
29
30
31#define CACHELINE_BYTES 64
32#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(u32))
33
34#define ENGINE_TRACE(e, fmt, ...) do { \
35 const struct intel_engine_cs *e__ __maybe_unused = (e); \
36 GEM_TRACE("%s %s: " fmt, \
37 dev_name(e__->i915->drm.dev), e__->name, \
38 ##__VA_ARGS__); \
39} while (0)
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54#define __ENGINE_REG_OP(op__, engine__, ...) \
55 intel_uncore_##op__((engine__)->uncore, __VA_ARGS__)
56
57#define __ENGINE_READ_OP(op__, engine__, reg__) \
58 __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base))
59
60#define ENGINE_READ16(...) __ENGINE_READ_OP(read16, __VA_ARGS__)
61#define ENGINE_READ(...) __ENGINE_READ_OP(read, __VA_ARGS__)
62#define ENGINE_READ_FW(...) __ENGINE_READ_OP(read_fw, __VA_ARGS__)
63#define ENGINE_POSTING_READ(...) __ENGINE_READ_OP(posting_read_fw, __VA_ARGS__)
64#define ENGINE_POSTING_READ16(...) __ENGINE_READ_OP(posting_read16, __VA_ARGS__)
65
66#define ENGINE_READ64(engine__, lower_reg__, upper_reg__) \
67 __ENGINE_REG_OP(read64_2x32, (engine__), \
68 lower_reg__((engine__)->mmio_base), \
69 upper_reg__((engine__)->mmio_base))
70
71#define ENGINE_READ_IDX(engine__, reg__, idx__) \
72 __ENGINE_REG_OP(read, (engine__), reg__((engine__)->mmio_base, (idx__)))
73
74#define __ENGINE_WRITE_OP(op__, engine__, reg__, val__) \
75 __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base), (val__))
76
77#define ENGINE_WRITE16(...) __ENGINE_WRITE_OP(write16, __VA_ARGS__)
78#define ENGINE_WRITE(...) __ENGINE_WRITE_OP(write, __VA_ARGS__)
79#define ENGINE_WRITE_FW(...) __ENGINE_WRITE_OP(write_fw, __VA_ARGS__)
80
81#define GEN6_RING_FAULT_REG_READ(engine__) \
82 intel_uncore_read((engine__)->uncore, RING_FAULT_REG(engine__))
83
84#define GEN6_RING_FAULT_REG_POSTING_READ(engine__) \
85 intel_uncore_posting_read((engine__)->uncore, RING_FAULT_REG(engine__))
86
87#define GEN6_RING_FAULT_REG_RMW(engine__, clear__, set__) \
88({ \
89 u32 __val; \
90\
91 __val = intel_uncore_read((engine__)->uncore, \
92 RING_FAULT_REG(engine__)); \
93 __val &= ~(clear__); \
94 __val |= (set__); \
95 intel_uncore_write((engine__)->uncore, RING_FAULT_REG(engine__), \
96 __val); \
97})
98
99
100
101
102
103static inline unsigned int
104execlists_num_ports(const struct intel_engine_execlists * const execlists)
105{
106 return execlists->port_mask + 1;
107}
108
109static inline struct i915_request *
110execlists_active(const struct intel_engine_execlists *execlists)
111{
112 struct i915_request * const *cur, * const *old, *active;
113
114 cur = READ_ONCE(execlists->active);
115 smp_rmb();
116 do {
117 old = cur;
118
119 active = READ_ONCE(*cur);
120 cur = READ_ONCE(execlists->active);
121
122 smp_rmb();
123 } while (unlikely(cur != old));
124
125 return active;
126}
127
128struct i915_request *
129execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists);
130
131static inline u32
132intel_read_status_page(const struct intel_engine_cs *engine, int reg)
133{
134
135 return READ_ONCE(engine->status_page.addr[reg]);
136}
137
138static inline void
139intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
140{
141
142
143
144
145
146 if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
147 mb();
148 clflush(&engine->status_page.addr[reg]);
149 engine->status_page.addr[reg] = value;
150 clflush(&engine->status_page.addr[reg]);
151 mb();
152 } else {
153 WRITE_ONCE(engine->status_page.addr[reg], value);
154 }
155}
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173#define I915_GEM_HWS_PREEMPT 0x32
174#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32))
175#define I915_GEM_HWS_SEQNO 0x40
176#define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32))
177#define I915_GEM_HWS_MIGRATE (0x42 * sizeof(u32))
178#define I915_GEM_HWS_SCRATCH 0x80
179
180#define I915_HWS_CSB_BUF0_INDEX 0x10
181#define I915_HWS_CSB_WRITE_INDEX 0x1f
182#define ICL_HWS_CSB_WRITE_INDEX 0x2f
183
184void intel_engine_stop(struct intel_engine_cs *engine);
185void intel_engine_cleanup(struct intel_engine_cs *engine);
186
187int intel_engines_init_mmio(struct intel_gt *gt);
188int intel_engines_init(struct intel_gt *gt);
189
190void intel_engine_free_request_pool(struct intel_engine_cs *engine);
191
192void intel_engines_release(struct intel_gt *gt);
193void intel_engines_free(struct intel_gt *gt);
194
195int intel_engine_init_common(struct intel_engine_cs *engine);
196void intel_engine_cleanup_common(struct intel_engine_cs *engine);
197
198int intel_engine_resume(struct intel_engine_cs *engine);
199
200int intel_ring_submission_setup(struct intel_engine_cs *engine);
201
202int intel_engine_stop_cs(struct intel_engine_cs *engine);
203void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine);
204
205void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask);
206
207u64 intel_engine_get_active_head(const struct intel_engine_cs *engine);
208u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine);
209
210void intel_engine_get_instdone(const struct intel_engine_cs *engine,
211 struct intel_instdone *instdone);
212
213void intel_engine_init_execlists(struct intel_engine_cs *engine);
214
215bool intel_engine_irq_enable(struct intel_engine_cs *engine);
216void intel_engine_irq_disable(struct intel_engine_cs *engine);
217
218static inline void __intel_engine_reset(struct intel_engine_cs *engine,
219 bool stalled)
220{
221 if (engine->reset.rewind)
222 engine->reset.rewind(engine, stalled);
223 engine->serial++;
224}
225
226bool intel_engines_are_idle(struct intel_gt *gt);
227bool intel_engine_is_idle(struct intel_engine_cs *engine);
228
229void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync);
230static inline void intel_engine_flush_submission(struct intel_engine_cs *engine)
231{
232 __intel_engine_flush_submission(engine, true);
233}
234
235void intel_engines_reset_default_submission(struct intel_gt *gt);
236
237bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
238
239__printf(3, 4)
240void intel_engine_dump(struct intel_engine_cs *engine,
241 struct drm_printer *m,
242 const char *header, ...);
243void intel_engine_dump_active_requests(struct list_head *requests,
244 struct i915_request *hung_rq,
245 struct drm_printer *m);
246
247ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine,
248 ktime_t *now);
249
250struct i915_request *
251intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine);
252
253u32 intel_engine_context_size(struct intel_gt *gt, u8 class);
254struct intel_context *
255intel_engine_create_pinned_context(struct intel_engine_cs *engine,
256 struct i915_address_space *vm,
257 unsigned int ring_size,
258 unsigned int hwsp,
259 struct lock_class_key *key,
260 const char *name);
261
262void intel_engine_destroy_pinned_context(struct intel_context *ce);
263
264#define ENGINE_PHYSICAL 0
265#define ENGINE_MOCK 1
266#define ENGINE_VIRTUAL 2
267
268static inline bool intel_engine_uses_guc(const struct intel_engine_cs *engine)
269{
270 return engine->gt->submission_method >= INTEL_SUBMISSION_GUC;
271}
272
273static inline bool
274intel_engine_has_preempt_reset(const struct intel_engine_cs *engine)
275{
276 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
277 return false;
278
279 return intel_engine_has_preemption(engine);
280}
281
282struct intel_context *
283intel_engine_create_virtual(struct intel_engine_cs **siblings,
284 unsigned int count);
285
286static inline bool
287intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine)
288{
289
290
291
292
293
294
295 GEM_BUG_ON(!intel_engine_uses_guc(engine));
296
297 return intel_guc_virtual_engine_has_heartbeat(engine);
298}
299
300static inline bool
301intel_engine_has_heartbeat(const struct intel_engine_cs *engine)
302{
303 if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL))
304 return false;
305
306 if (intel_engine_is_virtual(engine))
307 return intel_virtual_engine_has_heartbeat(engine);
308 else
309 return READ_ONCE(engine->props.heartbeat_interval_ms);
310}
311
312static inline struct intel_engine_cs *
313intel_engine_get_sibling(struct intel_engine_cs *engine, unsigned int sibling)
314{
315 GEM_BUG_ON(!intel_engine_is_virtual(engine));
316 return engine->cops->get_sibling(engine, sibling);
317}
318
319static inline void
320intel_engine_set_hung_context(struct intel_engine_cs *engine,
321 struct intel_context *ce)
322{
323 engine->hung_ce = ce;
324}
325
326static inline void
327intel_engine_clear_hung_context(struct intel_engine_cs *engine)
328{
329 intel_engine_set_hung_context(engine, NULL);
330}
331
332static inline struct intel_context *
333intel_engine_get_hung_context(struct intel_engine_cs *engine)
334{
335 return engine->hung_ce;
336}
337
338#endif
339