1
2
3
4
5
6#include <linux/dma-fence-array.h>
7#include <linux/reservation.h>
8#include <linux/sync_file.h>
9#include "etnaviv_cmdbuf.h"
10#include "etnaviv_drv.h"
11#include "etnaviv_gpu.h"
12#include "etnaviv_gem.h"
13#include "etnaviv_perfmon.h"
14#include "etnaviv_sched.h"
15
16
17
18
19
20#define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE)
21
22#define BO_LOCKED 0x4000
23#define BO_PINNED 0x2000
24
25static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
26 struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs)
27{
28 struct etnaviv_gem_submit *submit;
29 size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit));
30
31 submit = kzalloc(sz, GFP_KERNEL);
32 if (!submit)
33 return NULL;
34
35 submit->pmrs = kcalloc(nr_pmrs, sizeof(struct etnaviv_perfmon_request),
36 GFP_KERNEL);
37 if (!submit->pmrs) {
38 kfree(submit);
39 return NULL;
40 }
41 submit->nr_pmrs = nr_pmrs;
42
43 submit->gpu = gpu;
44 kref_init(&submit->refcount);
45
46 return submit;
47}
48
49static int submit_lookup_objects(struct etnaviv_gem_submit *submit,
50 struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos,
51 unsigned nr_bos)
52{
53 struct drm_etnaviv_gem_submit_bo *bo;
54 unsigned i;
55 int ret = 0;
56
57 spin_lock(&file->table_lock);
58
59 for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) {
60 struct drm_gem_object *obj;
61
62 if (bo->flags & BO_INVALID_FLAGS) {
63 DRM_ERROR("invalid flags: %x\n", bo->flags);
64 ret = -EINVAL;
65 goto out_unlock;
66 }
67
68 submit->bos[i].flags = bo->flags;
69
70
71
72
73 obj = idr_find(&file->object_idr, bo->handle);
74 if (!obj) {
75 DRM_ERROR("invalid handle %u at index %u\n",
76 bo->handle, i);
77 ret = -EINVAL;
78 goto out_unlock;
79 }
80
81
82
83
84
85 drm_gem_object_get(obj);
86
87 submit->bos[i].obj = to_etnaviv_bo(obj);
88 }
89
90out_unlock:
91 submit->nr_bos = i;
92 spin_unlock(&file->table_lock);
93
94 return ret;
95}
96
97static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)
98{
99 if (submit->bos[i].flags & BO_LOCKED) {
100 struct drm_gem_object *obj = &submit->bos[i].obj->base;
101
102 ww_mutex_unlock(&obj->resv->lock);
103 submit->bos[i].flags &= ~BO_LOCKED;
104 }
105}
106
107static int submit_lock_objects(struct etnaviv_gem_submit *submit,
108 struct ww_acquire_ctx *ticket)
109{
110 int contended, slow_locked = -1, i, ret = 0;
111
112retry:
113 for (i = 0; i < submit->nr_bos; i++) {
114 struct drm_gem_object *obj = &submit->bos[i].obj->base;
115
116 if (slow_locked == i)
117 slow_locked = -1;
118
119 contended = i;
120
121 if (!(submit->bos[i].flags & BO_LOCKED)) {
122 ret = ww_mutex_lock_interruptible(&obj->resv->lock,
123 ticket);
124 if (ret == -EALREADY)
125 DRM_ERROR("BO at index %u already on submit list\n",
126 i);
127 if (ret)
128 goto fail;
129 submit->bos[i].flags |= BO_LOCKED;
130 }
131 }
132
133 ww_acquire_done(ticket);
134
135 return 0;
136
137fail:
138 for (; i >= 0; i--)
139 submit_unlock_object(submit, i);
140
141 if (slow_locked > 0)
142 submit_unlock_object(submit, slow_locked);
143
144 if (ret == -EDEADLK) {
145 struct drm_gem_object *obj;
146
147 obj = &submit->bos[contended].obj->base;
148
149
150 ret = ww_mutex_lock_slow_interruptible(&obj->resv->lock,
151 ticket);
152 if (!ret) {
153 submit->bos[contended].flags |= BO_LOCKED;
154 slow_locked = contended;
155 goto retry;
156 }
157 }
158
159 return ret;
160}
161
162static int submit_fence_sync(struct etnaviv_gem_submit *submit)
163{
164 int i, ret = 0;
165
166 for (i = 0; i < submit->nr_bos; i++) {
167 struct etnaviv_gem_submit_bo *bo = &submit->bos[i];
168 struct reservation_object *robj = bo->obj->base.resv;
169
170 if (!(bo->flags & ETNA_SUBMIT_BO_WRITE)) {
171 ret = reservation_object_reserve_shared(robj, 1);
172 if (ret)
173 return ret;
174 }
175
176 if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT)
177 continue;
178
179 if (bo->flags & ETNA_SUBMIT_BO_WRITE) {
180 ret = reservation_object_get_fences_rcu(robj, &bo->excl,
181 &bo->nr_shared,
182 &bo->shared);
183 if (ret)
184 return ret;
185 } else {
186 bo->excl = reservation_object_get_excl_rcu(robj);
187 }
188
189 }
190
191 return ret;
192}
193
194static void submit_attach_object_fences(struct etnaviv_gem_submit *submit)
195{
196 int i;
197
198 for (i = 0; i < submit->nr_bos; i++) {
199 struct drm_gem_object *obj = &submit->bos[i].obj->base;
200
201 if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE)
202 reservation_object_add_excl_fence(obj->resv,
203 submit->out_fence);
204 else
205 reservation_object_add_shared_fence(obj->resv,
206 submit->out_fence);
207
208 submit_unlock_object(submit, i);
209 }
210}
211
212static int submit_pin_objects(struct etnaviv_gem_submit *submit)
213{
214 int i, ret = 0;
215
216 for (i = 0; i < submit->nr_bos; i++) {
217 struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
218 struct etnaviv_vram_mapping *mapping;
219
220 mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base,
221 submit->gpu);
222 if (IS_ERR(mapping)) {
223 ret = PTR_ERR(mapping);
224 break;
225 }
226 atomic_inc(&etnaviv_obj->gpu_active);
227
228 submit->bos[i].flags |= BO_PINNED;
229 submit->bos[i].mapping = mapping;
230 }
231
232 return ret;
233}
234
235static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx,
236 struct etnaviv_gem_submit_bo **bo)
237{
238 if (idx >= submit->nr_bos) {
239 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
240 idx, submit->nr_bos);
241 return -EINVAL;
242 }
243
244 *bo = &submit->bos[idx];
245
246 return 0;
247}
248
249
250static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream,
251 u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs,
252 u32 nr_relocs)
253{
254 u32 i, last_offset = 0;
255 u32 *ptr = stream;
256 int ret;
257
258 for (i = 0; i < nr_relocs; i++) {
259 const struct drm_etnaviv_gem_submit_reloc *r = relocs + i;
260 struct etnaviv_gem_submit_bo *bo;
261 u32 off;
262
263 if (unlikely(r->flags)) {
264 DRM_ERROR("invalid reloc flags\n");
265 return -EINVAL;
266 }
267
268 if (r->submit_offset % 4) {
269 DRM_ERROR("non-aligned reloc offset: %u\n",
270 r->submit_offset);
271 return -EINVAL;
272 }
273
274
275 off = r->submit_offset / 4;
276
277 if ((off >= size ) ||
278 (off < last_offset)) {
279 DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
280 return -EINVAL;
281 }
282
283 ret = submit_bo(submit, r->reloc_idx, &bo);
284 if (ret)
285 return ret;
286
287 if (r->reloc_offset > bo->obj->base.size - sizeof(*ptr)) {
288 DRM_ERROR("relocation %u outside object\n", i);
289 return -EINVAL;
290 }
291
292 ptr[off] = bo->mapping->iova + r->reloc_offset;
293
294 last_offset = off;
295 }
296
297 return 0;
298}
299
300static int submit_perfmon_validate(struct etnaviv_gem_submit *submit,
301 u32 exec_state, const struct drm_etnaviv_gem_submit_pmr *pmrs)
302{
303 u32 i;
304
305 for (i = 0; i < submit->nr_pmrs; i++) {
306 const struct drm_etnaviv_gem_submit_pmr *r = pmrs + i;
307 struct etnaviv_gem_submit_bo *bo;
308 int ret;
309
310 ret = submit_bo(submit, r->read_idx, &bo);
311 if (ret)
312 return ret;
313
314
315 if (r->read_offset == 0) {
316 DRM_ERROR("perfmon request: offset is 0");
317 return -EINVAL;
318 }
319
320 if (r->read_offset >= bo->obj->base.size - sizeof(u32)) {
321 DRM_ERROR("perfmon request: offset %u outside object", i);
322 return -EINVAL;
323 }
324
325 if (r->flags & ~(ETNA_PM_PROCESS_PRE | ETNA_PM_PROCESS_POST)) {
326 DRM_ERROR("perfmon request: flags are not valid");
327 return -EINVAL;
328 }
329
330 if (etnaviv_pm_req_validate(r, exec_state)) {
331 DRM_ERROR("perfmon request: domain or signal not valid");
332 return -EINVAL;
333 }
334
335 submit->pmrs[i].flags = r->flags;
336 submit->pmrs[i].domain = r->domain;
337 submit->pmrs[i].signal = r->signal;
338 submit->pmrs[i].sequence = r->sequence;
339 submit->pmrs[i].offset = r->read_offset;
340 submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base);
341 }
342
343 return 0;
344}
345
346static void submit_cleanup(struct kref *kref)
347{
348 struct etnaviv_gem_submit *submit =
349 container_of(kref, struct etnaviv_gem_submit, refcount);
350 unsigned i;
351
352 if (submit->runtime_resumed)
353 pm_runtime_put_autosuspend(submit->gpu->dev);
354
355 if (submit->cmdbuf.suballoc)
356 etnaviv_cmdbuf_free(&submit->cmdbuf);
357
358 for (i = 0; i < submit->nr_bos; i++) {
359 struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
360
361
362 if (submit->bos[i].flags & BO_PINNED) {
363 etnaviv_gem_mapping_unreference(submit->bos[i].mapping);
364 atomic_dec(&etnaviv_obj->gpu_active);
365 submit->bos[i].mapping = NULL;
366 submit->bos[i].flags &= ~BO_PINNED;
367 }
368
369
370 submit_unlock_object(submit, i);
371 drm_gem_object_put_unlocked(&etnaviv_obj->base);
372 }
373
374 wake_up_all(&submit->gpu->fence_event);
375
376 if (submit->in_fence)
377 dma_fence_put(submit->in_fence);
378 if (submit->out_fence) {
379
380 mutex_lock(&submit->gpu->fence_lock);
381 idr_remove(&submit->gpu->fence_idr, submit->out_fence_id);
382 mutex_unlock(&submit->gpu->fence_lock);
383 dma_fence_put(submit->out_fence);
384 }
385 kfree(submit->pmrs);
386 kfree(submit);
387}
388
389void etnaviv_submit_put(struct etnaviv_gem_submit *submit)
390{
391 kref_put(&submit->refcount, submit_cleanup);
392}
393
394int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
395 struct drm_file *file)
396{
397 struct etnaviv_file_private *ctx = file->driver_priv;
398 struct etnaviv_drm_private *priv = dev->dev_private;
399 struct drm_etnaviv_gem_submit *args = data;
400 struct drm_etnaviv_gem_submit_reloc *relocs;
401 struct drm_etnaviv_gem_submit_pmr *pmrs;
402 struct drm_etnaviv_gem_submit_bo *bos;
403 struct etnaviv_gem_submit *submit;
404 struct etnaviv_gpu *gpu;
405 struct sync_file *sync_file = NULL;
406 struct ww_acquire_ctx ticket;
407 int out_fence_fd = -1;
408 void *stream;
409 int ret;
410
411 if (args->pipe >= ETNA_MAX_PIPES)
412 return -EINVAL;
413
414 gpu = priv->gpu[args->pipe];
415 if (!gpu)
416 return -ENXIO;
417
418 if (args->stream_size % 4) {
419 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
420 args->stream_size);
421 return -EINVAL;
422 }
423
424 if (args->exec_state != ETNA_PIPE_3D &&
425 args->exec_state != ETNA_PIPE_2D &&
426 args->exec_state != ETNA_PIPE_VG) {
427 DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state);
428 return -EINVAL;
429 }
430
431 if (args->flags & ~ETNA_SUBMIT_FLAGS) {
432 DRM_ERROR("invalid flags: 0x%x\n", args->flags);
433 return -EINVAL;
434 }
435
436
437
438
439
440 bos = kvmalloc_array(args->nr_bos, sizeof(*bos), GFP_KERNEL);
441 relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL);
442 pmrs = kvmalloc_array(args->nr_pmrs, sizeof(*pmrs), GFP_KERNEL);
443 stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL);
444 if (!bos || !relocs || !pmrs || !stream) {
445 ret = -ENOMEM;
446 goto err_submit_cmds;
447 }
448
449 ret = copy_from_user(bos, u64_to_user_ptr(args->bos),
450 args->nr_bos * sizeof(*bos));
451 if (ret) {
452 ret = -EFAULT;
453 goto err_submit_cmds;
454 }
455
456 ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs),
457 args->nr_relocs * sizeof(*relocs));
458 if (ret) {
459 ret = -EFAULT;
460 goto err_submit_cmds;
461 }
462
463 ret = copy_from_user(pmrs, u64_to_user_ptr(args->pmrs),
464 args->nr_pmrs * sizeof(*pmrs));
465 if (ret) {
466 ret = -EFAULT;
467 goto err_submit_cmds;
468 }
469
470 ret = copy_from_user(stream, u64_to_user_ptr(args->stream),
471 args->stream_size);
472 if (ret) {
473 ret = -EFAULT;
474 goto err_submit_cmds;
475 }
476
477 if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) {
478 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
479 if (out_fence_fd < 0) {
480 ret = out_fence_fd;
481 goto err_submit_cmds;
482 }
483 }
484
485 ww_acquire_init(&ticket, &reservation_ww_class);
486
487 submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs);
488 if (!submit) {
489 ret = -ENOMEM;
490 goto err_submit_ww_acquire;
491 }
492
493 ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &submit->cmdbuf,
494 ALIGN(args->stream_size, 8) + 8);
495 if (ret)
496 goto err_submit_objects;
497
498 submit->ctx = file->driver_priv;
499 submit->exec_state = args->exec_state;
500 submit->flags = args->flags;
501
502 ret = submit_lookup_objects(submit, file, bos, args->nr_bos);
503 if (ret)
504 goto err_submit_objects;
505
506 if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4,
507 relocs, args->nr_relocs)) {
508 ret = -EINVAL;
509 goto err_submit_objects;
510 }
511
512 if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) {
513 submit->in_fence = sync_file_get_fence(args->fence_fd);
514 if (!submit->in_fence) {
515 ret = -EINVAL;
516 goto err_submit_objects;
517 }
518 }
519
520 ret = submit_pin_objects(submit);
521 if (ret)
522 goto err_submit_objects;
523
524 ret = submit_reloc(submit, stream, args->stream_size / 4,
525 relocs, args->nr_relocs);
526 if (ret)
527 goto err_submit_objects;
528
529 ret = submit_perfmon_validate(submit, args->exec_state, pmrs);
530 if (ret)
531 goto err_submit_objects;
532
533 memcpy(submit->cmdbuf.vaddr, stream, args->stream_size);
534
535 ret = submit_lock_objects(submit, &ticket);
536 if (ret)
537 goto err_submit_objects;
538
539 ret = submit_fence_sync(submit);
540 if (ret)
541 goto err_submit_objects;
542
543 ret = etnaviv_sched_push_job(&ctx->sched_entity[args->pipe], submit);
544 if (ret)
545 goto err_submit_objects;
546
547 submit_attach_object_fences(submit);
548
549 if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) {
550
551
552
553
554
555
556 sync_file = sync_file_create(submit->out_fence);
557 if (!sync_file) {
558 ret = -ENOMEM;
559 goto err_submit_objects;
560 }
561 fd_install(out_fence_fd, sync_file->file);
562 }
563
564 args->fence_fd = out_fence_fd;
565 args->fence = submit->out_fence_id;
566
567err_submit_objects:
568 etnaviv_submit_put(submit);
569
570err_submit_ww_acquire:
571 ww_acquire_fini(&ticket);
572
573err_submit_cmds:
574 if (ret && (out_fence_fd >= 0))
575 put_unused_fd(out_fence_fd);
576 if (stream)
577 kvfree(stream);
578 if (bos)
579 kvfree(bos);
580 if (relocs)
581 kvfree(relocs);
582 if (pmrs)
583 kvfree(pmrs);
584
585 return ret;
586}
587
