1
2
3
4#include <linux/delay.h>
5#include <linux/mutex.h>
6#include <linux/spinlock_types.h>
7#include <linux/workqueue.h>
8
9#include <drm/drm_encoder.h>
10#include <drm/drm_gem.h>
11#include <drm/drm_gem_shmem_helper.h>
12#include <drm/gpu_scheduler.h>
13
14#include "uapi/drm/v3d_drm.h"
15
16struct clk;
17struct platform_device;
18struct reset_control;
19
20#define GMP_GRANULARITY (128 * 1024)
21
22
23enum v3d_queue {
24 V3D_BIN,
25 V3D_RENDER,
26 V3D_TFU,
27 V3D_CSD,
28 V3D_CACHE_CLEAN,
29};
30
31#define V3D_MAX_QUEUES (V3D_CACHE_CLEAN + 1)
32
33struct v3d_queue_state {
34 struct drm_gpu_scheduler sched;
35
36 u64 fence_context;
37 u64 emit_seqno;
38};
39
40
41
42
43
44
45
46struct v3d_perfmon {
47
48
49
50 refcount_t refcnt;
51
52
53 struct mutex lock;
54
55
56
57
58 u8 ncounters;
59
60
61 u8 counters[DRM_V3D_MAX_PERF_COUNTERS];
62
63
64
65
66
67
68
69
70
71 u64 values[];
72};
73
74struct v3d_dev {
75 struct drm_device drm;
76
77
78
79
80 int ver;
81 bool single_irq_line;
82
83 void __iomem *hub_regs;
84 void __iomem *core_regs[3];
85 void __iomem *bridge_regs;
86 void __iomem *gca_regs;
87 struct clk *clk;
88 struct reset_control *reset;
89
90
91 volatile u32 *pt;
92 dma_addr_t pt_paddr;
93
94
95
96
97
98 void *mmu_scratch;
99 dma_addr_t mmu_scratch_paddr;
100
101 int va_width;
102
103
104 u32 cores;
105
106
107
108
109 struct drm_mm mm;
110 spinlock_t mm_lock;
111
112 struct work_struct overflow_mem_work;
113
114 struct v3d_bin_job *bin_job;
115 struct v3d_render_job *render_job;
116 struct v3d_tfu_job *tfu_job;
117 struct v3d_csd_job *csd_job;
118
119 struct v3d_queue_state queue[V3D_MAX_QUEUES];
120
121
122
123
124 spinlock_t job_lock;
125
126
127 struct v3d_perfmon *active_perfmon;
128
129
130 struct mutex bo_lock;
131
132
133
134
135
136 struct mutex reset_lock;
137
138
139
140
141 struct mutex sched_lock;
142
143
144
145
146
147 struct mutex cache_clean_lock;
148
149 struct {
150 u32 num_allocated;
151 u32 pages_allocated;
152 } bo_stats;
153};
154
155static inline struct v3d_dev *
156to_v3d_dev(struct drm_device *dev)
157{
158 return container_of(dev, struct v3d_dev, drm);
159}
160
161static inline bool
162v3d_has_csd(struct v3d_dev *v3d)
163{
164 return v3d->ver >= 41;
165}
166
167#define v3d_to_pdev(v3d) to_platform_device((v3d)->drm.dev)
168
169
170struct v3d_file_priv {
171 struct v3d_dev *v3d;
172
173 struct {
174 struct idr idr;
175 struct mutex lock;
176 } perfmon;
177
178 struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
179};
180
181struct v3d_bo {
182 struct drm_gem_shmem_object base;
183
184 struct drm_mm_node node;
185
186
187
188
189 struct list_head unref_head;
190};
191
192static inline struct v3d_bo *
193to_v3d_bo(struct drm_gem_object *bo)
194{
195 return (struct v3d_bo *)bo;
196}
197
198struct v3d_fence {
199 struct dma_fence base;
200 struct drm_device *dev;
201
202 u64 seqno;
203 enum v3d_queue queue;
204};
205
206static inline struct v3d_fence *
207to_v3d_fence(struct dma_fence *fence)
208{
209 return (struct v3d_fence *)fence;
210}
211
212#define V3D_READ(offset) readl(v3d->hub_regs + offset)
213#define V3D_WRITE(offset, val) writel(val, v3d->hub_regs + offset)
214
215#define V3D_BRIDGE_READ(offset) readl(v3d->bridge_regs + offset)
216#define V3D_BRIDGE_WRITE(offset, val) writel(val, v3d->bridge_regs + offset)
217
218#define V3D_GCA_READ(offset) readl(v3d->gca_regs + offset)
219#define V3D_GCA_WRITE(offset, val) writel(val, v3d->gca_regs + offset)
220
221#define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset)
222#define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset)
223
224struct v3d_job {
225 struct drm_sched_job base;
226
227 struct kref refcount;
228
229 struct v3d_dev *v3d;
230
231
232
233
234 struct drm_gem_object **bo;
235 u32 bo_count;
236
237
238
239 struct xarray deps;
240 unsigned long last_dep;
241
242
243 struct dma_fence *irq_fence;
244
245
246
247
248 struct dma_fence *done_fence;
249
250
251
252
253 struct v3d_perfmon *perfmon;
254
255
256 void (*free)(struct kref *ref);
257};
258
259struct v3d_bin_job {
260 struct v3d_job base;
261
262
263 u32 start, end;
264
265 u32 timedout_ctca, timedout_ctra;
266
267
268 struct v3d_render_job *render;
269
270
271 u32 qma, qms, qts;
272};
273
274struct v3d_render_job {
275 struct v3d_job base;
276
277
278 u32 start, end;
279
280 u32 timedout_ctca, timedout_ctra;
281
282
283
284
285 struct list_head unref_list;
286};
287
288struct v3d_tfu_job {
289 struct v3d_job base;
290
291 struct drm_v3d_submit_tfu args;
292};
293
294struct v3d_csd_job {
295 struct v3d_job base;
296
297 u32 timedout_batches;
298
299 struct drm_v3d_submit_csd args;
300};
301
302
303
304
305
306
307
308
309
310#define __wait_for(OP, COND, US, Wmin, Wmax) ({ \
311 const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (US)); \
312 long wait__ = (Wmin); \
313 int ret__; \
314 might_sleep(); \
315 for (;;) { \
316 const bool expired__ = ktime_after(ktime_get_raw(), end__); \
317 OP; \
318 \
319 barrier(); \
320 if (COND) { \
321 ret__ = 0; \
322 break; \
323 } \
324 if (expired__) { \
325 ret__ = -ETIMEDOUT; \
326 break; \
327 } \
328 usleep_range(wait__, wait__ * 2); \
329 if (wait__ < (Wmax)) \
330 wait__ <<= 1; \
331 } \
332 ret__; \
333})
334
335#define _wait_for(COND, US, Wmin, Wmax) __wait_for(, (COND), (US), (Wmin), \
336 (Wmax))
337#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)
338
339static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
340{
341
342 if (NSEC_PER_SEC % HZ &&
343 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
344 return MAX_JIFFY_OFFSET;
345
346 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
347}
348
349
350struct drm_gem_object *v3d_create_object(struct drm_device *dev, size_t size);
351void v3d_free_object(struct drm_gem_object *gem_obj);
352struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
353 size_t size);
354int v3d_create_bo_ioctl(struct drm_device *dev, void *data,
355 struct drm_file *file_priv);
356int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
357 struct drm_file *file_priv);
358int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
359 struct drm_file *file_priv);
360struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
361 struct dma_buf_attachment *attach,
362 struct sg_table *sgt);
363
364
365void v3d_debugfs_init(struct drm_minor *minor);
366
367
368extern const struct dma_fence_ops v3d_fence_ops;
369struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue queue);
370
371
372int v3d_gem_init(struct drm_device *dev);
373void v3d_gem_destroy(struct drm_device *dev);
374int v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
375 struct drm_file *file_priv);
376int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
377 struct drm_file *file_priv);
378int v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
379 struct drm_file *file_priv);
380int v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
381 struct drm_file *file_priv);
382void v3d_job_put(struct v3d_job *job);
383void v3d_reset(struct v3d_dev *v3d);
384void v3d_invalidate_caches(struct v3d_dev *v3d);
385void v3d_clean_caches(struct v3d_dev *v3d);
386
387
388int v3d_irq_init(struct v3d_dev *v3d);
389void v3d_irq_enable(struct v3d_dev *v3d);
390void v3d_irq_disable(struct v3d_dev *v3d);
391void v3d_irq_reset(struct v3d_dev *v3d);
392
393
394int v3d_mmu_get_offset(struct drm_file *file_priv, struct v3d_bo *bo,
395 u32 *offset);
396int v3d_mmu_set_page_table(struct v3d_dev *v3d);
397void v3d_mmu_insert_ptes(struct v3d_bo *bo);
398void v3d_mmu_remove_ptes(struct v3d_bo *bo);
399
400
401int v3d_sched_init(struct v3d_dev *v3d);
402void v3d_sched_fini(struct v3d_dev *v3d);
403
404
405void v3d_perfmon_get(struct v3d_perfmon *perfmon);
406void v3d_perfmon_put(struct v3d_perfmon *perfmon);
407void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon);
408void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
409 bool capture);
410struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id);
411void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv);
412void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv);
413int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
414 struct drm_file *file_priv);
415int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
416 struct drm_file *file_priv);
417int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
418 struct drm_file *file_priv);
419