1
2
3
4
5
6
7#ifndef __MSM_GPU_H__
8#define __MSM_GPU_H__
9
10#include <linux/adreno-smmu-priv.h>
11#include <linux/clk.h>
12#include <linux/interconnect.h>
13#include <linux/pm_opp.h>
14#include <linux/regulator/consumer.h>
15
16#include "msm_drv.h"
17#include "msm_fence.h"
18#include "msm_ringbuffer.h"
19#include "msm_gem.h"
20
21struct msm_gem_submit;
22struct msm_gpu_perfcntr;
23struct msm_gpu_state;
24
25struct msm_gpu_config {
26 const char *ioname;
27 unsigned int nr_rings;
28};
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44struct msm_gpu_funcs {
45 int (*get_param)(struct msm_gpu *gpu, uint32_t param, uint64_t *value);
46 int (*hw_init)(struct msm_gpu *gpu);
47 int (*pm_suspend)(struct msm_gpu *gpu);
48 int (*pm_resume)(struct msm_gpu *gpu);
49 void (*submit)(struct msm_gpu *gpu, struct msm_gem_submit *submit);
50 void (*flush)(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
51 irqreturn_t (*irq)(struct msm_gpu *irq);
52 struct msm_ringbuffer *(*active_ring)(struct msm_gpu *gpu);
53 void (*recover)(struct msm_gpu *gpu);
54 void (*destroy)(struct msm_gpu *gpu);
55#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
56
57 void (*show)(struct msm_gpu *gpu, struct msm_gpu_state *state,
58 struct drm_printer *p);
59
60 void (*debugfs_init)(struct msm_gpu *gpu, struct drm_minor *minor);
61#endif
62 unsigned long (*gpu_busy)(struct msm_gpu *gpu);
63 struct msm_gpu_state *(*gpu_state_get)(struct msm_gpu *gpu);
64 int (*gpu_state_put)(struct msm_gpu_state *state);
65 unsigned long (*gpu_get_freq)(struct msm_gpu *gpu);
66 void (*gpu_set_freq)(struct msm_gpu *gpu, struct dev_pm_opp *opp);
67 struct msm_gem_address_space *(*create_address_space)
68 (struct msm_gpu *gpu, struct platform_device *pdev);
69 struct msm_gem_address_space *(*create_private_address_space)
70 (struct msm_gpu *gpu);
71 uint32_t (*get_rptr)(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
72};
73
74
75struct msm_gpu_fault_info {
76 u64 ttbr0;
77 unsigned long iova;
78 int flags;
79 const char *type;
80 const char *block;
81};
82
83
84
85
86struct msm_gpu_devfreq {
87
88 struct devfreq *devfreq;
89
90
91
92
93
94
95
96
97 u64 busy_cycles;
98
99
100 ktime_t time;
101
102
103 ktime_t idle_time;
104
105
106
107
108
109
110
111
112
113
114 unsigned long idle_freq;
115};
116
117struct msm_gpu {
118 const char *name;
119 struct drm_device *dev;
120 struct platform_device *pdev;
121 const struct msm_gpu_funcs *funcs;
122
123 struct adreno_smmu_priv adreno_smmu;
124
125
126 spinlock_t perf_lock;
127 bool perfcntr_active;
128 struct {
129 bool active;
130 ktime_t time;
131 } last_sample;
132 uint32_t totaltime, activetime;
133 uint32_t last_cntrs[5];
134 const struct msm_gpu_perfcntr *perfcntrs;
135 uint32_t num_perfcntrs;
136
137 struct msm_ringbuffer *rb[MSM_GPU_MAX_RINGS];
138 int nr_rings;
139
140
141
142
143
144 struct list_head active_list;
145
146
147
148
149
150
151
152
153
154 int active_submits;
155
156
157 struct mutex active_lock;
158
159
160 bool needs_hw_init;
161
162
163 int global_faults;
164
165 void __iomem *mmio;
166 int irq;
167
168 struct msm_gem_address_space *aspace;
169
170
171 struct regulator *gpu_reg, *gpu_cx;
172 struct clk_bulk_data *grp_clks;
173 int nr_clocks;
174 struct clk *ebi1_clk, *core_clk, *rbbmtimer_clk;
175 uint32_t fast_rate;
176
177
178
179#define DRM_MSM_INACTIVE_PERIOD 66
180
181#define DRM_MSM_HANGCHECK_DEFAULT_PERIOD 500
182 struct timer_list hangcheck_timer;
183
184
185 struct msm_gpu_fault_info fault_info;
186
187
188 struct kthread_work fault_work;
189
190
191 struct kthread_work recover_work;
192
193
194 struct kthread_work retire_work;
195
196
197 struct kthread_worker *worker;
198
199 struct drm_gem_object *memptrs_bo;
200
201 struct msm_gpu_devfreq devfreq;
202
203 uint32_t suspend_count;
204
205 struct msm_gpu_state *crashstate;
206
207
208 bool clamp_to_idle;
209
210
211 bool hw_apriv;
212
213 struct thermal_cooling_device *cooling;
214};
215
216static inline struct msm_gpu *dev_to_gpu(struct device *dev)
217{
218 struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev);
219 return container_of(adreno_smmu, struct msm_gpu, adreno_smmu);
220}
221
222
223#define MSM_GPU_RINGBUFFER_SZ SZ_32K
224#define MSM_GPU_RINGBUFFER_BLKSIZE 32
225
226#define MSM_GPU_RB_CNTL_DEFAULT \
227 (AXXX_CP_RB_CNTL_BUFSZ(ilog2(MSM_GPU_RINGBUFFER_SZ / 8)) | \
228 AXXX_CP_RB_CNTL_BLKSZ(ilog2(MSM_GPU_RINGBUFFER_BLKSIZE / 8)))
229
230static inline bool msm_gpu_active(struct msm_gpu *gpu)
231{
232 int i;
233
234 for (i = 0; i < gpu->nr_rings; i++) {
235 struct msm_ringbuffer *ring = gpu->rb[i];
236
237 if (ring->seqno > ring->memptrs->fence)
238 return true;
239 }
240
241 return false;
242}
243
244
245
246
247
248
249
250struct msm_gpu_perfcntr {
251 uint32_t select_reg;
252 uint32_t sample_reg;
253 uint32_t select_val;
254 const char *name;
255};
256
257
258
259
260
261
262#define NR_SCHED_PRIORITIES (1 + DRM_SCHED_PRIORITY_HIGH - DRM_SCHED_PRIORITY_MIN)
263
264
265
266
267
268
269
270
271
272
273
274
275struct msm_file_private {
276 rwlock_t queuelock;
277 struct list_head submitqueues;
278 int queueid;
279 struct msm_gem_address_space *aspace;
280 struct kref ref;
281 int seqno;
282
283
284
285
286
287
288
289
290
291
292
293
294 struct drm_sched_entity *entities[NR_SCHED_PRIORITIES * MSM_GPU_MAX_RINGS];
295};
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322static inline int msm_gpu_convert_priority(struct msm_gpu *gpu, int prio,
323 unsigned *ring_nr, enum drm_sched_priority *sched_prio)
324{
325 unsigned rn, sp;
326
327 rn = div_u64_rem(prio, NR_SCHED_PRIORITIES, &sp);
328
329
330
331
332 sp = NR_SCHED_PRIORITIES - sp - 1;
333
334 if (rn >= gpu->nr_rings)
335 return -EINVAL;
336
337 *ring_nr = rn;
338 *sched_prio = sp;
339
340 return 0;
341}
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365struct msm_gpu_submitqueue {
366 int id;
367 u32 flags;
368 u32 ring_nr;
369 int faults;
370 struct msm_file_private *ctx;
371 struct list_head node;
372 struct idr fence_idr;
373 struct mutex lock;
374 struct kref ref;
375 struct drm_sched_entity *entity;
376};
377
378struct msm_gpu_state_bo {
379 u64 iova;
380 size_t size;
381 void *data;
382 bool encoded;
383};
384
385struct msm_gpu_state {
386 struct kref ref;
387 struct timespec64 time;
388
389 struct {
390 u64 iova;
391 u32 fence;
392 u32 seqno;
393 u32 rptr;
394 u32 wptr;
395 void *data;
396 int data_size;
397 bool encoded;
398 } ring[MSM_GPU_MAX_RINGS];
399
400 int nr_registers;
401 u32 *registers;
402
403 u32 rbbm_status;
404
405 char *comm;
406 char *cmd;
407
408 struct msm_gpu_fault_info fault_info;
409
410 int nr_bos;
411 struct msm_gpu_state_bo *bos;
412};
413
414static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
415{
416 msm_writel(data, gpu->mmio + (reg << 2));
417}
418
419static inline u32 gpu_read(struct msm_gpu *gpu, u32 reg)
420{
421 return msm_readl(gpu->mmio + (reg << 2));
422}
423
424static inline void gpu_rmw(struct msm_gpu *gpu, u32 reg, u32 mask, u32 or)
425{
426 msm_rmw(gpu->mmio + (reg << 2), mask, or);
427}
428
429static inline u64 gpu_read64(struct msm_gpu *gpu, u32 lo, u32 hi)
430{
431 u64 val;
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447 val = (u64) msm_readl(gpu->mmio + (lo << 2));
448 val |= ((u64) msm_readl(gpu->mmio + (hi << 2)) << 32);
449
450 return val;
451}
452
453static inline void gpu_write64(struct msm_gpu *gpu, u32 lo, u32 hi, u64 val)
454{
455
456 msm_writel(lower_32_bits(val), gpu->mmio + (lo << 2));
457 msm_writel(upper_32_bits(val), gpu->mmio + (hi << 2));
458}
459
460int msm_gpu_pm_suspend(struct msm_gpu *gpu);
461int msm_gpu_pm_resume(struct msm_gpu *gpu);
462
463int msm_submitqueue_init(struct drm_device *drm, struct msm_file_private *ctx);
464struct msm_gpu_submitqueue *msm_submitqueue_get(struct msm_file_private *ctx,
465 u32 id);
466int msm_submitqueue_create(struct drm_device *drm,
467 struct msm_file_private *ctx,
468 u32 prio, u32 flags, u32 *id);
469int msm_submitqueue_query(struct drm_device *drm, struct msm_file_private *ctx,
470 struct drm_msm_submitqueue_query *args);
471int msm_submitqueue_remove(struct msm_file_private *ctx, u32 id);
472void msm_submitqueue_close(struct msm_file_private *ctx);
473
474void msm_submitqueue_destroy(struct kref *kref);
475
476void __msm_file_private_destroy(struct kref *kref);
477
478static inline void msm_file_private_put(struct msm_file_private *ctx)
479{
480 kref_put(&ctx->ref, __msm_file_private_destroy);
481}
482
483static inline struct msm_file_private *msm_file_private_get(
484 struct msm_file_private *ctx)
485{
486 kref_get(&ctx->ref);
487 return ctx;
488}
489
490void msm_devfreq_init(struct msm_gpu *gpu);
491void msm_devfreq_cleanup(struct msm_gpu *gpu);
492void msm_devfreq_resume(struct msm_gpu *gpu);
493void msm_devfreq_suspend(struct msm_gpu *gpu);
494void msm_devfreq_active(struct msm_gpu *gpu);
495void msm_devfreq_idle(struct msm_gpu *gpu);
496
497int msm_gpu_hw_init(struct msm_gpu *gpu);
498
499void msm_gpu_perfcntr_start(struct msm_gpu *gpu);
500void msm_gpu_perfcntr_stop(struct msm_gpu *gpu);
501int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,
502 uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs);
503
504void msm_gpu_retire(struct msm_gpu *gpu);
505void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit);
506
507int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
508 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
509 const char *name, struct msm_gpu_config *config);
510
511struct msm_gem_address_space *
512msm_gpu_create_private_address_space(struct msm_gpu *gpu, struct task_struct *task);
513
514void msm_gpu_cleanup(struct msm_gpu *gpu);
515
516struct msm_gpu *adreno_load_gpu(struct drm_device *dev);
517void __init adreno_register(void);
518void __exit adreno_unregister(void);
519
520static inline void msm_submitqueue_put(struct msm_gpu_submitqueue *queue)
521{
522 if (queue)
523 kref_put(&queue->ref, msm_submitqueue_destroy);
524}
525
526static inline struct msm_gpu_state *msm_gpu_crashstate_get(struct msm_gpu *gpu)
527{
528 struct msm_gpu_state *state = NULL;
529
530 mutex_lock(&gpu->dev->struct_mutex);
531
532 if (gpu->crashstate) {
533 kref_get(&gpu->crashstate->ref);
534 state = gpu->crashstate;
535 }
536
537 mutex_unlock(&gpu->dev->struct_mutex);
538
539 return state;
540}
541
542static inline void msm_gpu_crashstate_put(struct msm_gpu *gpu)
543{
544 mutex_lock(&gpu->dev->struct_mutex);
545
546 if (gpu->crashstate) {
547 if (gpu->funcs->gpu_state_put(gpu->crashstate))
548 gpu->crashstate = NULL;
549 }
550
551 mutex_unlock(&gpu->dev->struct_mutex);
552}
553
554
555
556
557
558#define check_apriv(gpu, flags) \
559 (((gpu)->hw_apriv ? MSM_BO_MAP_PRIV : 0) | (flags))
560
561
562#endif
563