1
2
3
4
5
6
7#include <linux/file.h>
8#include <linux/sync_file.h>
9#include <linux/uaccess.h>
10
11#include <drm/drm_drv.h>
12#include <drm/drm_file.h>
13#include <drm/drm_syncobj.h>
14
15#include "msm_drv.h"
16#include "msm_gpu.h"
17#include "msm_gem.h"
18#include "msm_gpu_trace.h"
19
20
21
22
23
24static struct msm_gem_submit *submit_create(struct drm_device *dev,
25 struct msm_gpu *gpu,
26 struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
27 uint32_t nr_cmds)
28{
29 struct msm_gem_submit *submit;
30 uint64_t sz;
31 int ret;
32
33 sz = struct_size(submit, bos, nr_bos) +
34 ((u64)nr_cmds * sizeof(submit->cmd[0]));
35
36 if (sz > SIZE_MAX)
37 return ERR_PTR(-ENOMEM);
38
39 submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
40 if (!submit)
41 return ERR_PTR(-ENOMEM);
42
43 ret = drm_sched_job_init(&submit->base, queue->entity, queue);
44 if (ret) {
45 kfree(submit);
46 return ERR_PTR(ret);
47 }
48
49 kref_init(&submit->ref);
50 submit->dev = dev;
51 submit->aspace = queue->ctx->aspace;
52 submit->gpu = gpu;
53 submit->cmd = (void *)&submit->bos[nr_bos];
54 submit->queue = queue;
55 submit->ring = gpu->rb[queue->ring_nr];
56 submit->fault_dumped = false;
57
58 INIT_LIST_HEAD(&submit->node);
59
60 return submit;
61}
62
63void __msm_gem_submit_destroy(struct kref *kref)
64{
65 struct msm_gem_submit *submit =
66 container_of(kref, struct msm_gem_submit, ref);
67 unsigned i;
68
69 if (submit->fence_id) {
70 mutex_lock(&submit->queue->lock);
71 idr_remove(&submit->queue->fence_idr, submit->fence_id);
72 mutex_unlock(&submit->queue->lock);
73 }
74
75 dma_fence_put(submit->user_fence);
76 dma_fence_put(submit->hw_fence);
77
78 put_pid(submit->pid);
79 msm_submitqueue_put(submit->queue);
80
81 for (i = 0; i < submit->nr_cmds; i++)
82 kfree(submit->cmd[i].relocs);
83
84 kfree(submit);
85}
86
87static int submit_lookup_objects(struct msm_gem_submit *submit,
88 struct drm_msm_gem_submit *args, struct drm_file *file)
89{
90 unsigned i;
91 int ret = 0;
92
93 for (i = 0; i < args->nr_bos; i++) {
94 struct drm_msm_gem_submit_bo submit_bo;
95 void __user *userptr =
96 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
97
98
99
100
101 submit->bos[i].flags = 0;
102
103 if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
104 ret = -EFAULT;
105 i = 0;
106 goto out;
107 }
108
109
110#define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
111
112 if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
113 !(submit_bo.flags & MANDATORY_FLAGS)) {
114 DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
115 ret = -EINVAL;
116 i = 0;
117 goto out;
118 }
119
120 submit->bos[i].handle = submit_bo.handle;
121 submit->bos[i].flags = submit_bo.flags;
122
123 submit->bos[i].iova = submit_bo.presumed;
124 }
125
126 spin_lock(&file->table_lock);
127
128 for (i = 0; i < args->nr_bos; i++) {
129 struct drm_gem_object *obj;
130
131
132
133
134 obj = idr_find(&file->object_idr, submit->bos[i].handle);
135 if (!obj) {
136 DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);
137 ret = -EINVAL;
138 goto out_unlock;
139 }
140
141 drm_gem_object_get(obj);
142
143 submit->bos[i].obj = to_msm_bo(obj);
144 }
145
146out_unlock:
147 spin_unlock(&file->table_lock);
148
149out:
150 submit->nr_bos = i;
151
152 return ret;
153}
154
155static int submit_lookup_cmds(struct msm_gem_submit *submit,
156 struct drm_msm_gem_submit *args, struct drm_file *file)
157{
158 unsigned i;
159 size_t sz;
160 int ret = 0;
161
162 for (i = 0; i < args->nr_cmds; i++) {
163 struct drm_msm_gem_submit_cmd submit_cmd;
164 void __user *userptr =
165 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
166
167 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
168 if (ret) {
169 ret = -EFAULT;
170 goto out;
171 }
172
173
174 switch (submit_cmd.type) {
175 case MSM_SUBMIT_CMD_BUF:
176 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
177 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
178 break;
179 default:
180 DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
181 return -EINVAL;
182 }
183
184 if (submit_cmd.size % 4) {
185 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
186 submit_cmd.size);
187 ret = -EINVAL;
188 goto out;
189 }
190
191 submit->cmd[i].type = submit_cmd.type;
192 submit->cmd[i].size = submit_cmd.size / 4;
193 submit->cmd[i].offset = submit_cmd.submit_offset / 4;
194 submit->cmd[i].idx = submit_cmd.submit_idx;
195 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
196
197 userptr = u64_to_user_ptr(submit_cmd.relocs);
198
199 sz = array_size(submit_cmd.nr_relocs,
200 sizeof(struct drm_msm_gem_submit_reloc));
201
202 if (sz == SIZE_MAX) {
203 ret = -ENOMEM;
204 goto out;
205 }
206 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL);
207 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
208 if (ret) {
209 ret = -EFAULT;
210 goto out;
211 }
212 }
213
214out:
215 return ret;
216}
217
218
219
220
221
222static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,
223 unsigned cleanup_flags)
224{
225 struct drm_gem_object *obj = &submit->bos[i].obj->base;
226 unsigned flags = submit->bos[i].flags & cleanup_flags;
227
228
229
230
231
232
233 submit->bos[i].flags &= ~cleanup_flags;
234
235 if (flags & BO_VMA_PINNED)
236 msm_gem_unpin_vma(submit->bos[i].vma);
237
238 if (flags & BO_OBJ_PINNED)
239 msm_gem_unpin_locked(obj);
240
241 if (flags & BO_ACTIVE)
242 msm_gem_active_put(obj);
243
244 if (flags & BO_LOCKED)
245 dma_resv_unlock(obj->resv);
246}
247
248static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
249{
250 unsigned cleanup_flags = BO_VMA_PINNED | BO_OBJ_PINNED |
251 BO_ACTIVE | BO_LOCKED;
252 submit_cleanup_bo(submit, i, cleanup_flags);
253
254 if (!(submit->bos[i].flags & BO_VALID))
255 submit->bos[i].iova = 0;
256}
257
258
259static int submit_lock_objects(struct msm_gem_submit *submit)
260{
261 int contended, slow_locked = -1, i, ret = 0;
262
263retry:
264 for (i = 0; i < submit->nr_bos; i++) {
265 struct msm_gem_object *msm_obj = submit->bos[i].obj;
266
267 if (slow_locked == i)
268 slow_locked = -1;
269
270 contended = i;
271
272 if (!(submit->bos[i].flags & BO_LOCKED)) {
273 ret = dma_resv_lock_interruptible(msm_obj->base.resv,
274 &submit->ticket);
275 if (ret)
276 goto fail;
277 submit->bos[i].flags |= BO_LOCKED;
278 }
279 }
280
281 ww_acquire_done(&submit->ticket);
282
283 return 0;
284
285fail:
286 if (ret == -EALREADY) {
287 DRM_ERROR("handle %u at index %u already on submit list\n",
288 submit->bos[i].handle, i);
289 ret = -EINVAL;
290 }
291
292 for (; i >= 0; i--)
293 submit_unlock_unpin_bo(submit, i);
294
295 if (slow_locked > 0)
296 submit_unlock_unpin_bo(submit, slow_locked);
297
298 if (ret == -EDEADLK) {
299 struct msm_gem_object *msm_obj = submit->bos[contended].obj;
300
301 ret = dma_resv_lock_slow_interruptible(msm_obj->base.resv,
302 &submit->ticket);
303 if (!ret) {
304 submit->bos[contended].flags |= BO_LOCKED;
305 slow_locked = contended;
306 goto retry;
307 }
308
309
310
311
312
313 WARN_ON_ONCE(ret == -EALREADY);
314 }
315
316 return ret;
317}
318
319static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
320{
321 int i, ret = 0;
322
323 for (i = 0; i < submit->nr_bos; i++) {
324 struct drm_gem_object *obj = &submit->bos[i].obj->base;
325 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
326
327
328
329
330
331
332 ret = dma_resv_reserve_fences(obj->resv, 1);
333 if (ret)
334 return ret;
335
336
337 if (no_implicit && !write)
338 continue;
339
340 ret = drm_sched_job_add_implicit_dependencies(&submit->base,
341 obj,
342 write);
343 if (ret)
344 break;
345 }
346
347 return ret;
348}
349
350static int submit_pin_objects(struct msm_gem_submit *submit)
351{
352 int i, ret = 0;
353
354 submit->valid = true;
355
356
357
358
359
360
361 for (i = 0; i < submit->nr_bos; i++) {
362 struct drm_gem_object *obj = &submit->bos[i].obj->base;
363
364 msm_gem_active_get(obj, submit->gpu);
365 submit->bos[i].flags |= BO_ACTIVE;
366 }
367
368 for (i = 0; i < submit->nr_bos; i++) {
369 struct drm_gem_object *obj = &submit->bos[i].obj->base;
370 struct msm_gem_vma *vma;
371
372
373 vma = msm_gem_get_vma_locked(obj, submit->aspace);
374 if (IS_ERR(vma)) {
375 ret = PTR_ERR(vma);
376 break;
377 }
378
379 ret = msm_gem_pin_vma_locked(obj, vma);
380 if (ret)
381 break;
382
383 submit->bos[i].flags |= BO_OBJ_PINNED | BO_VMA_PINNED;
384 submit->bos[i].vma = vma;
385
386 if (vma->iova == submit->bos[i].iova) {
387 submit->bos[i].flags |= BO_VALID;
388 } else {
389 submit->bos[i].iova = vma->iova;
390
391 submit->bos[i].flags &= ~BO_VALID;
392 submit->valid = false;
393 }
394 }
395
396 return ret;
397}
398
399static void submit_attach_object_fences(struct msm_gem_submit *submit)
400{
401 int i;
402
403 for (i = 0; i < submit->nr_bos; i++) {
404 struct drm_gem_object *obj = &submit->bos[i].obj->base;
405
406 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
407 dma_resv_add_fence(obj->resv, submit->user_fence,
408 DMA_RESV_USAGE_WRITE);
409 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
410 dma_resv_add_fence(obj->resv, submit->user_fence,
411 DMA_RESV_USAGE_READ);
412 }
413}
414
415static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
416 struct msm_gem_object **obj, uint64_t *iova, bool *valid)
417{
418 if (idx >= submit->nr_bos) {
419 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
420 idx, submit->nr_bos);
421 return -EINVAL;
422 }
423
424 if (obj)
425 *obj = submit->bos[idx].obj;
426 if (iova)
427 *iova = submit->bos[idx].iova;
428 if (valid)
429 *valid = !!(submit->bos[idx].flags & BO_VALID);
430
431 return 0;
432}
433
434
435static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
436 uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
437{
438 uint32_t i, last_offset = 0;
439 uint32_t *ptr;
440 int ret = 0;
441
442 if (!nr_relocs)
443 return 0;
444
445 if (offset % 4) {
446 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
447 return -EINVAL;
448 }
449
450
451
452
453 ptr = msm_gem_get_vaddr_locked(&obj->base);
454
455 if (IS_ERR(ptr)) {
456 ret = PTR_ERR(ptr);
457 DBG("failed to map: %d", ret);
458 return ret;
459 }
460
461 for (i = 0; i < nr_relocs; i++) {
462 struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
463 uint32_t off;
464 uint64_t iova;
465 bool valid;
466
467 if (submit_reloc.submit_offset % 4) {
468 DRM_ERROR("non-aligned reloc offset: %u\n",
469 submit_reloc.submit_offset);
470 ret = -EINVAL;
471 goto out;
472 }
473
474
475 off = submit_reloc.submit_offset / 4;
476
477 if ((off >= (obj->base.size / 4)) ||
478 (off < last_offset)) {
479 DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
480 ret = -EINVAL;
481 goto out;
482 }
483
484 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
485 if (ret)
486 goto out;
487
488 if (valid)
489 continue;
490
491 iova += submit_reloc.reloc_offset;
492
493 if (submit_reloc.shift < 0)
494 iova >>= -submit_reloc.shift;
495 else
496 iova <<= submit_reloc.shift;
497
498 ptr[off] = iova | submit_reloc.or;
499
500 last_offset = off;
501 }
502
503out:
504 msm_gem_put_vaddr_locked(&obj->base);
505
506 return ret;
507}
508
509
510
511
512
513static void submit_cleanup(struct msm_gem_submit *submit, bool error)
514{
515 unsigned cleanup_flags = BO_LOCKED;
516 unsigned i;
517
518 if (error)
519 cleanup_flags |= BO_VMA_PINNED | BO_OBJ_PINNED | BO_ACTIVE;
520
521 for (i = 0; i < submit->nr_bos; i++) {
522 struct msm_gem_object *msm_obj = submit->bos[i].obj;
523 submit_cleanup_bo(submit, i, cleanup_flags);
524 if (error)
525 drm_gem_object_put(&msm_obj->base);
526 }
527}
528
529void msm_submit_retire(struct msm_gem_submit *submit)
530{
531 int i;
532
533 for (i = 0; i < submit->nr_bos; i++) {
534 struct drm_gem_object *obj = &submit->bos[i].obj->base;
535
536 msm_gem_lock(obj);
537
538 submit_cleanup_bo(submit, i, BO_OBJ_PINNED | BO_ACTIVE);
539 msm_gem_unlock(obj);
540 drm_gem_object_put(obj);
541 }
542}
543
544struct msm_submit_post_dep {
545 struct drm_syncobj *syncobj;
546 uint64_t point;
547 struct dma_fence_chain *chain;
548};
549
550static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
551 struct drm_file *file,
552 uint64_t in_syncobjs_addr,
553 uint32_t nr_in_syncobjs,
554 size_t syncobj_stride,
555 struct msm_ringbuffer *ring)
556{
557 struct drm_syncobj **syncobjs = NULL;
558 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
559 int ret = 0;
560 uint32_t i, j;
561
562 syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
563 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
564 if (!syncobjs)
565 return ERR_PTR(-ENOMEM);
566
567 for (i = 0; i < nr_in_syncobjs; ++i) {
568 uint64_t address = in_syncobjs_addr + i * syncobj_stride;
569 struct dma_fence *fence;
570
571 if (copy_from_user(&syncobj_desc,
572 u64_to_user_ptr(address),
573 min(syncobj_stride, sizeof(syncobj_desc)))) {
574 ret = -EFAULT;
575 break;
576 }
577
578 if (syncobj_desc.point &&
579 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) {
580 ret = -EOPNOTSUPP;
581 break;
582 }
583
584 if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
585 ret = -EINVAL;
586 break;
587 }
588
589 ret = drm_syncobj_find_fence(file, syncobj_desc.handle,
590 syncobj_desc.point, 0, &fence);
591 if (ret)
592 break;
593
594 ret = drm_sched_job_add_dependency(&submit->base, fence);
595 if (ret)
596 break;
597
598 if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
599 syncobjs[i] =
600 drm_syncobj_find(file, syncobj_desc.handle);
601 if (!syncobjs[i]) {
602 ret = -EINVAL;
603 break;
604 }
605 }
606 }
607
608 if (ret) {
609 for (j = 0; j <= i; ++j) {
610 if (syncobjs[j])
611 drm_syncobj_put(syncobjs[j]);
612 }
613 kfree(syncobjs);
614 return ERR_PTR(ret);
615 }
616 return syncobjs;
617}
618
619static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
620 uint32_t nr_syncobjs)
621{
622 uint32_t i;
623
624 for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
625 if (syncobjs[i])
626 drm_syncobj_replace_fence(syncobjs[i], NULL);
627 }
628}
629
630static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
631 struct drm_file *file,
632 uint64_t syncobjs_addr,
633 uint32_t nr_syncobjs,
634 size_t syncobj_stride)
635{
636 struct msm_submit_post_dep *post_deps;
637 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
638 int ret = 0;
639 uint32_t i, j;
640
641 post_deps = kmalloc_array(nr_syncobjs, sizeof(*post_deps),
642 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
643 if (!post_deps)
644 return ERR_PTR(-ENOMEM);
645
646 for (i = 0; i < nr_syncobjs; ++i) {
647 uint64_t address = syncobjs_addr + i * syncobj_stride;
648
649 if (copy_from_user(&syncobj_desc,
650 u64_to_user_ptr(address),
651 min(syncobj_stride, sizeof(syncobj_desc)))) {
652 ret = -EFAULT;
653 break;
654 }
655
656 post_deps[i].point = syncobj_desc.point;
657 post_deps[i].chain = NULL;
658
659 if (syncobj_desc.flags) {
660 ret = -EINVAL;
661 break;
662 }
663
664 if (syncobj_desc.point) {
665 if (!drm_core_check_feature(dev,
666 DRIVER_SYNCOBJ_TIMELINE)) {
667 ret = -EOPNOTSUPP;
668 break;
669 }
670
671 post_deps[i].chain = dma_fence_chain_alloc();
672 if (!post_deps[i].chain) {
673 ret = -ENOMEM;
674 break;
675 }
676 }
677
678 post_deps[i].syncobj =
679 drm_syncobj_find(file, syncobj_desc.handle);
680 if (!post_deps[i].syncobj) {
681 ret = -EINVAL;
682 break;
683 }
684 }
685
686 if (ret) {
687 for (j = 0; j <= i; ++j) {
688 dma_fence_chain_free(post_deps[j].chain);
689 if (post_deps[j].syncobj)
690 drm_syncobj_put(post_deps[j].syncobj);
691 }
692
693 kfree(post_deps);
694 return ERR_PTR(ret);
695 }
696
697 return post_deps;
698}
699
700static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
701 uint32_t count, struct dma_fence *fence)
702{
703 uint32_t i;
704
705 for (i = 0; post_deps && i < count; ++i) {
706 if (post_deps[i].chain) {
707 drm_syncobj_add_point(post_deps[i].syncobj,
708 post_deps[i].chain,
709 fence, post_deps[i].point);
710 post_deps[i].chain = NULL;
711 } else {
712 drm_syncobj_replace_fence(post_deps[i].syncobj,
713 fence);
714 }
715 }
716}
717
718int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
719 struct drm_file *file)
720{
721 static atomic_t ident = ATOMIC_INIT(0);
722 struct msm_drm_private *priv = dev->dev_private;
723 struct drm_msm_gem_submit *args = data;
724 struct msm_file_private *ctx = file->driver_priv;
725 struct msm_gem_submit *submit = NULL;
726 struct msm_gpu *gpu = priv->gpu;
727 struct msm_gpu_submitqueue *queue;
728 struct msm_ringbuffer *ring;
729 struct msm_submit_post_dep *post_deps = NULL;
730 struct drm_syncobj **syncobjs_to_reset = NULL;
731 int out_fence_fd = -1;
732 struct pid *pid = get_pid(task_pid(current));
733 bool has_ww_ticket = false;
734 unsigned i;
735 int ret, submitid;
736
737 if (!gpu)
738 return -ENXIO;
739
740 if (args->pad)
741 return -EINVAL;
742
743 if (unlikely(!ctx->aspace) && !capable(CAP_SYS_RAWIO)) {
744 DRM_ERROR_RATELIMITED("IOMMU support or CAP_SYS_RAWIO required!\n");
745 return -EPERM;
746 }
747
748
749
750
751 if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
752 return -EINVAL;
753
754 if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
755 return -EINVAL;
756
757 if (args->flags & MSM_SUBMIT_SUDO) {
758 if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
759 !capable(CAP_SYS_RAWIO))
760 return -EINVAL;
761 }
762
763 queue = msm_submitqueue_get(ctx, args->queueid);
764 if (!queue)
765 return -ENOENT;
766
767
768 submitid = atomic_inc_return(&ident) - 1;
769
770 ring = gpu->rb[queue->ring_nr];
771 trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid,
772 args->nr_bos, args->nr_cmds);
773
774 ret = mutex_lock_interruptible(&queue->lock);
775 if (ret)
776 goto out_post_unlock;
777
778 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
779 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
780 if (out_fence_fd < 0) {
781 ret = out_fence_fd;
782 goto out_unlock;
783 }
784 }
785
786 submit = submit_create(dev, gpu, queue, args->nr_bos,
787 args->nr_cmds);
788 if (IS_ERR(submit)) {
789 ret = PTR_ERR(submit);
790 submit = NULL;
791 goto out_unlock;
792 }
793
794 submit->pid = pid;
795 submit->ident = submitid;
796
797 if (args->flags & MSM_SUBMIT_SUDO)
798 submit->in_rb = true;
799
800 if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
801 struct dma_fence *in_fence;
802
803 in_fence = sync_file_get_fence(args->fence_fd);
804
805 if (!in_fence) {
806 ret = -EINVAL;
807 goto out_unlock;
808 }
809
810 ret = drm_sched_job_add_dependency(&submit->base, in_fence);
811 if (ret)
812 goto out_unlock;
813 }
814
815 if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
816 syncobjs_to_reset = msm_parse_deps(submit, file,
817 args->in_syncobjs,
818 args->nr_in_syncobjs,
819 args->syncobj_stride, ring);
820 if (IS_ERR(syncobjs_to_reset)) {
821 ret = PTR_ERR(syncobjs_to_reset);
822 goto out_unlock;
823 }
824 }
825
826 if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
827 post_deps = msm_parse_post_deps(dev, file,
828 args->out_syncobjs,
829 args->nr_out_syncobjs,
830 args->syncobj_stride);
831 if (IS_ERR(post_deps)) {
832 ret = PTR_ERR(post_deps);
833 goto out_unlock;
834 }
835 }
836
837 ret = submit_lookup_objects(submit, args, file);
838 if (ret)
839 goto out;
840
841 ret = submit_lookup_cmds(submit, args, file);
842 if (ret)
843 goto out;
844
845
846 ww_acquire_init(&submit->ticket, &reservation_ww_class);
847 has_ww_ticket = true;
848 ret = submit_lock_objects(submit);
849 if (ret)
850 goto out;
851
852 ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));
853 if (ret)
854 goto out;
855
856 ret = submit_pin_objects(submit);
857 if (ret)
858 goto out;
859
860 for (i = 0; i < args->nr_cmds; i++) {
861 struct msm_gem_object *msm_obj;
862 uint64_t iova;
863
864 ret = submit_bo(submit, submit->cmd[i].idx,
865 &msm_obj, &iova, NULL);
866 if (ret)
867 goto out;
868
869 if (!submit->cmd[i].size ||
870 ((submit->cmd[i].size + submit->cmd[i].offset) >
871 msm_obj->base.size / 4)) {
872 DRM_ERROR("invalid cmdstream size: %u\n", submit->cmd[i].size * 4);
873 ret = -EINVAL;
874 goto out;
875 }
876
877 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
878
879 if (submit->valid)
880 continue;
881
882 ret = submit_reloc(submit, msm_obj, submit->cmd[i].offset * 4,
883 submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
884 if (ret)
885 goto out;
886 }
887
888 submit->nr_cmds = i;
889
890
891
892
893
894
895
896 if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
897 idr_find(&queue->fence_idr, args->fence)) {
898 ret = -EINVAL;
899 goto out;
900 }
901
902 drm_sched_job_arm(&submit->base);
903
904 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);
905
906 if (args->flags & MSM_SUBMIT_FENCE_SN_IN) {
907
908
909
910
911
912 submit->fence_id = args->fence;
913 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
914 &submit->fence_id, submit->fence_id,
915 GFP_KERNEL);
916
917
918
919
920 WARN_ON(ret);
921 } else {
922
923
924
925
926 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
927 submit->user_fence, 1,
928 INT_MAX, GFP_KERNEL);
929 }
930 if (submit->fence_id < 0) {
931 ret = submit->fence_id;
932 submit->fence_id = 0;
933 }
934
935 if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
936 struct sync_file *sync_file = sync_file_create(submit->user_fence);
937 if (!sync_file) {
938 ret = -ENOMEM;
939 } else {
940 fd_install(out_fence_fd, sync_file->file);
941 args->fence_fd = out_fence_fd;
942 }
943 }
944
945 submit_attach_object_fences(submit);
946
947
948 msm_gem_submit_get(submit);
949
950 drm_sched_entity_push_job(&submit->base);
951
952 args->fence = submit->fence_id;
953 queue->last_fence = submit->fence_id;
954
955 msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
956 msm_process_post_deps(post_deps, args->nr_out_syncobjs,
957 submit->user_fence);
958
959
960out:
961 submit_cleanup(submit, !!ret);
962 if (has_ww_ticket)
963 ww_acquire_fini(&submit->ticket);
964out_unlock:
965 if (ret && (out_fence_fd >= 0))
966 put_unused_fd(out_fence_fd);
967 mutex_unlock(&queue->lock);
968 if (submit)
969 msm_gem_submit_put(submit);
970out_post_unlock:
971 if (!IS_ERR_OR_NULL(post_deps)) {
972 for (i = 0; i < args->nr_out_syncobjs; ++i) {
973 kfree(post_deps[i].chain);
974 drm_syncobj_put(post_deps[i].syncobj);
975 }
976 kfree(post_deps);
977 }
978
979 if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
980 for (i = 0; i < args->nr_in_syncobjs; ++i) {
981 if (syncobjs_to_reset[i])
982 drm_syncobj_put(syncobjs_to_reset[i]);
983 }
984 kfree(syncobjs_to_reset);
985 }
986
987 return ret;
988}
989