1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186#include <linux/anon_inodes.h>
187#include <linux/file.h>
188#include <linux/fs.h>
189#include <linux/sched/signal.h>
190#include <linux/sync_file.h>
191#include <linux/uaccess.h>
192
193#include <drm/drm.h>
194#include <drm/drm_drv.h>
195#include <drm/drm_file.h>
196#include <drm/drm_gem.h>
197#include <drm/drm_print.h>
198#include <drm/drm_syncobj.h>
199#include <drm/drm_utils.h>
200
201#include "drm_internal.h"
202
203struct syncobj_wait_entry {
204 struct list_head node;
205 struct task_struct *task;
206 struct dma_fence *fence;
207 struct dma_fence_cb fence_cb;
208 u64 point;
209};
210
211static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
212 struct syncobj_wait_entry *wait);
213
214
215
216
217
218
219
220
221
222struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
223 u32 handle)
224{
225 struct drm_syncobj *syncobj;
226
227 spin_lock(&file_private->syncobj_table_lock);
228
229
230 syncobj = idr_find(&file_private->syncobj_idr, handle);
231 if (syncobj)
232 drm_syncobj_get(syncobj);
233
234 spin_unlock(&file_private->syncobj_table_lock);
235
236 return syncobj;
237}
238EXPORT_SYMBOL(drm_syncobj_find);
239
240static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
241 struct syncobj_wait_entry *wait)
242{
243 struct dma_fence *fence;
244
245 if (wait->fence)
246 return;
247
248 spin_lock(&syncobj->lock);
249
250
251
252
253 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
254 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
255 dma_fence_put(fence);
256 list_add_tail(&wait->node, &syncobj->cb_list);
257 } else if (!fence) {
258 wait->fence = dma_fence_get_stub();
259 } else {
260 wait->fence = fence;
261 }
262 spin_unlock(&syncobj->lock);
263}
264
265static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
266 struct syncobj_wait_entry *wait)
267{
268 if (!wait->node.next)
269 return;
270
271 spin_lock(&syncobj->lock);
272 list_del_init(&wait->node);
273 spin_unlock(&syncobj->lock);
274}
275
276
277
278
279
280
281
282
283
284
285void drm_syncobj_add_point(struct drm_syncobj *syncobj,
286 struct dma_fence_chain *chain,
287 struct dma_fence *fence,
288 uint64_t point)
289{
290 struct syncobj_wait_entry *cur, *tmp;
291 struct dma_fence *prev;
292
293 dma_fence_get(fence);
294
295 spin_lock(&syncobj->lock);
296
297 prev = drm_syncobj_fence_get(syncobj);
298
299 if (prev && prev->seqno >= point)
300 DRM_ERROR("You are adding an unorder point to timeline!\n");
301 dma_fence_chain_init(chain, prev, fence, point);
302 rcu_assign_pointer(syncobj->fence, &chain->base);
303
304 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
305 syncobj_wait_syncobj_func(syncobj, cur);
306 spin_unlock(&syncobj->lock);
307
308
309 dma_fence_chain_for_each(fence, prev);
310 dma_fence_put(prev);
311}
312EXPORT_SYMBOL(drm_syncobj_add_point);
313
314
315
316
317
318
319
320
321void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
322 struct dma_fence *fence)
323{
324 struct dma_fence *old_fence;
325 struct syncobj_wait_entry *cur, *tmp;
326
327 if (fence)
328 dma_fence_get(fence);
329
330 spin_lock(&syncobj->lock);
331
332 old_fence = rcu_dereference_protected(syncobj->fence,
333 lockdep_is_held(&syncobj->lock));
334 rcu_assign_pointer(syncobj->fence, fence);
335
336 if (fence != old_fence) {
337 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
338 syncobj_wait_syncobj_func(syncobj, cur);
339 }
340
341 spin_unlock(&syncobj->lock);
342
343 dma_fence_put(old_fence);
344}
345EXPORT_SYMBOL(drm_syncobj_replace_fence);
346
347
348
349
350
351
352
353static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
354{
355 struct dma_fence *fence = dma_fence_get_stub();
356
357 drm_syncobj_replace_fence(syncobj, fence);
358 dma_fence_put(fence);
359}
360
361
362#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378int drm_syncobj_find_fence(struct drm_file *file_private,
379 u32 handle, u64 point, u64 flags,
380 struct dma_fence **fence)
381{
382 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
383 struct syncobj_wait_entry wait;
384 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
385 int ret;
386
387 if (!syncobj)
388 return -ENOENT;
389
390 *fence = drm_syncobj_fence_get(syncobj);
391 drm_syncobj_put(syncobj);
392
393 if (*fence) {
394 ret = dma_fence_chain_find_seqno(fence, point);
395 if (!ret)
396 return 0;
397 dma_fence_put(*fence);
398 } else {
399 ret = -EINVAL;
400 }
401
402 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
403 return ret;
404
405 memset(&wait, 0, sizeof(wait));
406 wait.task = current;
407 wait.point = point;
408 drm_syncobj_fence_add_wait(syncobj, &wait);
409
410 do {
411 set_current_state(TASK_INTERRUPTIBLE);
412 if (wait.fence) {
413 ret = 0;
414 break;
415 }
416 if (timeout == 0) {
417 ret = -ETIME;
418 break;
419 }
420
421 if (signal_pending(current)) {
422 ret = -ERESTARTSYS;
423 break;
424 }
425
426 timeout = schedule_timeout(timeout);
427 } while (1);
428
429 __set_current_state(TASK_RUNNING);
430 *fence = wait.fence;
431
432 if (wait.node.next)
433 drm_syncobj_remove_wait(syncobj, &wait);
434
435 return ret;
436}
437EXPORT_SYMBOL(drm_syncobj_find_fence);
438
439
440
441
442
443
444
445void drm_syncobj_free(struct kref *kref)
446{
447 struct drm_syncobj *syncobj = container_of(kref,
448 struct drm_syncobj,
449 refcount);
450 drm_syncobj_replace_fence(syncobj, NULL);
451 kfree(syncobj);
452}
453EXPORT_SYMBOL(drm_syncobj_free);
454
455
456
457
458
459
460
461
462
463
464
465
466
467int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
468 struct dma_fence *fence)
469{
470 struct drm_syncobj *syncobj;
471
472 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
473 if (!syncobj)
474 return -ENOMEM;
475
476 kref_init(&syncobj->refcount);
477 INIT_LIST_HEAD(&syncobj->cb_list);
478 spin_lock_init(&syncobj->lock);
479
480 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
481 drm_syncobj_assign_null_handle(syncobj);
482
483 if (fence)
484 drm_syncobj_replace_fence(syncobj, fence);
485
486 *out_syncobj = syncobj;
487 return 0;
488}
489EXPORT_SYMBOL(drm_syncobj_create);
490
491
492
493
494
495
496
497
498
499
500
501
502int drm_syncobj_get_handle(struct drm_file *file_private,
503 struct drm_syncobj *syncobj, u32 *handle)
504{
505 int ret;
506
507
508 drm_syncobj_get(syncobj);
509
510 idr_preload(GFP_KERNEL);
511 spin_lock(&file_private->syncobj_table_lock);
512 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
513 spin_unlock(&file_private->syncobj_table_lock);
514
515 idr_preload_end();
516
517 if (ret < 0) {
518 drm_syncobj_put(syncobj);
519 return ret;
520 }
521
522 *handle = ret;
523 return 0;
524}
525EXPORT_SYMBOL(drm_syncobj_get_handle);
526
527static int drm_syncobj_create_as_handle(struct drm_file *file_private,
528 u32 *handle, uint32_t flags)
529{
530 int ret;
531 struct drm_syncobj *syncobj;
532
533 ret = drm_syncobj_create(&syncobj, flags, NULL);
534 if (ret)
535 return ret;
536
537 ret = drm_syncobj_get_handle(file_private, syncobj, handle);
538 drm_syncobj_put(syncobj);
539 return ret;
540}
541
542static int drm_syncobj_destroy(struct drm_file *file_private,
543 u32 handle)
544{
545 struct drm_syncobj *syncobj;
546
547 spin_lock(&file_private->syncobj_table_lock);
548 syncobj = idr_remove(&file_private->syncobj_idr, handle);
549 spin_unlock(&file_private->syncobj_table_lock);
550
551 if (!syncobj)
552 return -EINVAL;
553
554 drm_syncobj_put(syncobj);
555 return 0;
556}
557
558static int drm_syncobj_file_release(struct inode *inode, struct file *file)
559{
560 struct drm_syncobj *syncobj = file->private_data;
561
562 drm_syncobj_put(syncobj);
563 return 0;
564}
565
566static const struct file_operations drm_syncobj_file_fops = {
567 .release = drm_syncobj_file_release,
568};
569
570
571
572
573
574
575
576
577
578
579int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
580{
581 struct file *file;
582 int fd;
583
584 fd = get_unused_fd_flags(O_CLOEXEC);
585 if (fd < 0)
586 return fd;
587
588 file = anon_inode_getfile("syncobj_file",
589 &drm_syncobj_file_fops,
590 syncobj, 0);
591 if (IS_ERR(file)) {
592 put_unused_fd(fd);
593 return PTR_ERR(file);
594 }
595
596 drm_syncobj_get(syncobj);
597 fd_install(fd, file);
598
599 *p_fd = fd;
600 return 0;
601}
602EXPORT_SYMBOL(drm_syncobj_get_fd);
603
604static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
605 u32 handle, int *p_fd)
606{
607 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
608 int ret;
609
610 if (!syncobj)
611 return -EINVAL;
612
613 ret = drm_syncobj_get_fd(syncobj, p_fd);
614 drm_syncobj_put(syncobj);
615 return ret;
616}
617
618static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
619 int fd, u32 *handle)
620{
621 struct drm_syncobj *syncobj;
622 struct fd f = fdget(fd);
623 int ret;
624
625 if (!f.file)
626 return -EINVAL;
627
628 if (f.file->f_op != &drm_syncobj_file_fops) {
629 fdput(f);
630 return -EINVAL;
631 }
632
633
634 syncobj = f.file->private_data;
635 drm_syncobj_get(syncobj);
636
637 idr_preload(GFP_KERNEL);
638 spin_lock(&file_private->syncobj_table_lock);
639 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
640 spin_unlock(&file_private->syncobj_table_lock);
641 idr_preload_end();
642
643 if (ret > 0) {
644 *handle = ret;
645 ret = 0;
646 } else
647 drm_syncobj_put(syncobj);
648
649 fdput(f);
650 return ret;
651}
652
653static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
654 int fd, int handle)
655{
656 struct dma_fence *fence = sync_file_get_fence(fd);
657 struct drm_syncobj *syncobj;
658
659 if (!fence)
660 return -EINVAL;
661
662 syncobj = drm_syncobj_find(file_private, handle);
663 if (!syncobj) {
664 dma_fence_put(fence);
665 return -ENOENT;
666 }
667
668 drm_syncobj_replace_fence(syncobj, fence);
669 dma_fence_put(fence);
670 drm_syncobj_put(syncobj);
671 return 0;
672}
673
674static int drm_syncobj_export_sync_file(struct drm_file *file_private,
675 int handle, int *p_fd)
676{
677 int ret;
678 struct dma_fence *fence;
679 struct sync_file *sync_file;
680 int fd = get_unused_fd_flags(O_CLOEXEC);
681
682 if (fd < 0)
683 return fd;
684
685 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
686 if (ret)
687 goto err_put_fd;
688
689 sync_file = sync_file_create(fence);
690
691 dma_fence_put(fence);
692
693 if (!sync_file) {
694 ret = -EINVAL;
695 goto err_put_fd;
696 }
697
698 fd_install(fd, sync_file->file);
699
700 *p_fd = fd;
701 return 0;
702err_put_fd:
703 put_unused_fd(fd);
704 return ret;
705}
706
707
708
709
710
711
712
713void
714drm_syncobj_open(struct drm_file *file_private)
715{
716 idr_init_base(&file_private->syncobj_idr, 1);
717 spin_lock_init(&file_private->syncobj_table_lock);
718}
719
720static int
721drm_syncobj_release_handle(int id, void *ptr, void *data)
722{
723 struct drm_syncobj *syncobj = ptr;
724
725 drm_syncobj_put(syncobj);
726 return 0;
727}
728
729
730
731
732
733
734
735
736
737void
738drm_syncobj_release(struct drm_file *file_private)
739{
740 idr_for_each(&file_private->syncobj_idr,
741 &drm_syncobj_release_handle, file_private);
742 idr_destroy(&file_private->syncobj_idr);
743}
744
745int
746drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
747 struct drm_file *file_private)
748{
749 struct drm_syncobj_create *args = data;
750
751 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
752 return -EOPNOTSUPP;
753
754
755 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
756 return -EINVAL;
757
758 return drm_syncobj_create_as_handle(file_private,
759 &args->handle, args->flags);
760}
761
762int
763drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
764 struct drm_file *file_private)
765{
766 struct drm_syncobj_destroy *args = data;
767
768 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
769 return -EOPNOTSUPP;
770
771
772 if (args->pad)
773 return -EINVAL;
774 return drm_syncobj_destroy(file_private, args->handle);
775}
776
777int
778drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
779 struct drm_file *file_private)
780{
781 struct drm_syncobj_handle *args = data;
782
783 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
784 return -EOPNOTSUPP;
785
786 if (args->pad)
787 return -EINVAL;
788
789 if (args->flags != 0 &&
790 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
791 return -EINVAL;
792
793 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
794 return drm_syncobj_export_sync_file(file_private, args->handle,
795 &args->fd);
796
797 return drm_syncobj_handle_to_fd(file_private, args->handle,
798 &args->fd);
799}
800
801int
802drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
803 struct drm_file *file_private)
804{
805 struct drm_syncobj_handle *args = data;
806
807 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
808 return -EOPNOTSUPP;
809
810 if (args->pad)
811 return -EINVAL;
812
813 if (args->flags != 0 &&
814 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
815 return -EINVAL;
816
817 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
818 return drm_syncobj_import_sync_file_fence(file_private,
819 args->fd,
820 args->handle);
821
822 return drm_syncobj_fd_to_handle(file_private, args->fd,
823 &args->handle);
824}
825
826static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
827 struct drm_syncobj_transfer *args)
828{
829 struct drm_syncobj *timeline_syncobj = NULL;
830 struct dma_fence *fence;
831 struct dma_fence_chain *chain;
832 int ret;
833
834 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
835 if (!timeline_syncobj) {
836 return -ENOENT;
837 }
838 ret = drm_syncobj_find_fence(file_private, args->src_handle,
839 args->src_point, args->flags,
840 &fence);
841 if (ret)
842 goto err;
843 chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
844 if (!chain) {
845 ret = -ENOMEM;
846 goto err1;
847 }
848 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
849err1:
850 dma_fence_put(fence);
851err:
852 drm_syncobj_put(timeline_syncobj);
853
854 return ret;
855}
856
857static int
858drm_syncobj_transfer_to_binary(struct drm_file *file_private,
859 struct drm_syncobj_transfer *args)
860{
861 struct drm_syncobj *binary_syncobj = NULL;
862 struct dma_fence *fence;
863 int ret;
864
865 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
866 if (!binary_syncobj)
867 return -ENOENT;
868 ret = drm_syncobj_find_fence(file_private, args->src_handle,
869 args->src_point, args->flags, &fence);
870 if (ret)
871 goto err;
872 drm_syncobj_replace_fence(binary_syncobj, fence);
873 dma_fence_put(fence);
874err:
875 drm_syncobj_put(binary_syncobj);
876
877 return ret;
878}
879int
880drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
881 struct drm_file *file_private)
882{
883 struct drm_syncobj_transfer *args = data;
884 int ret;
885
886 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
887 return -EOPNOTSUPP;
888
889 if (args->pad)
890 return -EINVAL;
891
892 if (args->dst_point)
893 ret = drm_syncobj_transfer_to_timeline(file_private, args);
894 else
895 ret = drm_syncobj_transfer_to_binary(file_private, args);
896
897 return ret;
898}
899
900static void syncobj_wait_fence_func(struct dma_fence *fence,
901 struct dma_fence_cb *cb)
902{
903 struct syncobj_wait_entry *wait =
904 container_of(cb, struct syncobj_wait_entry, fence_cb);
905
906 wake_up_process(wait->task);
907}
908
909static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
910 struct syncobj_wait_entry *wait)
911{
912 struct dma_fence *fence;
913
914
915 fence = rcu_dereference_protected(syncobj->fence,
916 lockdep_is_held(&syncobj->lock));
917 dma_fence_get(fence);
918 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
919 dma_fence_put(fence);
920 return;
921 } else if (!fence) {
922 wait->fence = dma_fence_get_stub();
923 } else {
924 wait->fence = fence;
925 }
926
927 wake_up_process(wait->task);
928 list_del_init(&wait->node);
929}
930
931static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
932 void __user *user_points,
933 uint32_t count,
934 uint32_t flags,
935 signed long timeout,
936 uint32_t *idx)
937{
938 struct syncobj_wait_entry *entries;
939 struct dma_fence *fence;
940 uint64_t *points;
941 uint32_t signaled_count, i;
942
943 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
944 if (points == NULL)
945 return -ENOMEM;
946
947 if (!user_points) {
948 memset(points, 0, count * sizeof(uint64_t));
949
950 } else if (copy_from_user(points, user_points,
951 sizeof(uint64_t) * count)) {
952 timeout = -EFAULT;
953 goto err_free_points;
954 }
955
956 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
957 if (!entries) {
958 timeout = -ENOMEM;
959 goto err_free_points;
960 }
961
962
963
964
965
966 signaled_count = 0;
967 for (i = 0; i < count; ++i) {
968 struct dma_fence *fence;
969
970 entries[i].task = current;
971 entries[i].point = points[i];
972 fence = drm_syncobj_fence_get(syncobjs[i]);
973 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
974 dma_fence_put(fence);
975 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
976 continue;
977 } else {
978 timeout = -EINVAL;
979 goto cleanup_entries;
980 }
981 }
982
983 if (fence)
984 entries[i].fence = fence;
985 else
986 entries[i].fence = dma_fence_get_stub();
987
988 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
989 dma_fence_is_signaled(entries[i].fence)) {
990 if (signaled_count == 0 && idx)
991 *idx = i;
992 signaled_count++;
993 }
994 }
995
996 if (signaled_count == count ||
997 (signaled_count > 0 &&
998 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
999 goto cleanup_entries;
1000
1001
1002
1003
1004
1005
1006
1007
1008 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1009 for (i = 0; i < count; ++i)
1010 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1011 }
1012
1013 do {
1014 set_current_state(TASK_INTERRUPTIBLE);
1015
1016 signaled_count = 0;
1017 for (i = 0; i < count; ++i) {
1018 fence = entries[i].fence;
1019 if (!fence)
1020 continue;
1021
1022 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1023 dma_fence_is_signaled(fence) ||
1024 (!entries[i].fence_cb.func &&
1025 dma_fence_add_callback(fence,
1026 &entries[i].fence_cb,
1027 syncobj_wait_fence_func))) {
1028
1029 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1030 signaled_count++;
1031 } else {
1032 if (idx)
1033 *idx = i;
1034 goto done_waiting;
1035 }
1036 }
1037 }
1038
1039 if (signaled_count == count)
1040 goto done_waiting;
1041
1042 if (timeout == 0) {
1043 timeout = -ETIME;
1044 goto done_waiting;
1045 }
1046
1047 if (signal_pending(current)) {
1048 timeout = -ERESTARTSYS;
1049 goto done_waiting;
1050 }
1051
1052 timeout = schedule_timeout(timeout);
1053 } while (1);
1054
1055done_waiting:
1056 __set_current_state(TASK_RUNNING);
1057
1058cleanup_entries:
1059 for (i = 0; i < count; ++i) {
1060 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1061 if (entries[i].fence_cb.func)
1062 dma_fence_remove_callback(entries[i].fence,
1063 &entries[i].fence_cb);
1064 dma_fence_put(entries[i].fence);
1065 }
1066 kfree(entries);
1067
1068err_free_points:
1069 kfree(points);
1070
1071 return timeout;
1072}
1073
1074
1075
1076
1077
1078
1079
1080
1081signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1082{
1083 ktime_t abs_timeout, now;
1084 u64 timeout_ns, timeout_jiffies64;
1085
1086
1087 if (timeout_nsec == 0)
1088 return 0;
1089
1090 abs_timeout = ns_to_ktime(timeout_nsec);
1091 now = ktime_get();
1092
1093 if (!ktime_after(abs_timeout, now))
1094 return 0;
1095
1096 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1097
1098 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1099
1100 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1101 return MAX_SCHEDULE_TIMEOUT - 1;
1102
1103 return timeout_jiffies64 + 1;
1104}
1105EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1106
1107static int drm_syncobj_array_wait(struct drm_device *dev,
1108 struct drm_file *file_private,
1109 struct drm_syncobj_wait *wait,
1110 struct drm_syncobj_timeline_wait *timeline_wait,
1111 struct drm_syncobj **syncobjs, bool timeline)
1112{
1113 signed long timeout = 0;
1114 uint32_t first = ~0;
1115
1116 if (!timeline) {
1117 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1118 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1119 NULL,
1120 wait->count_handles,
1121 wait->flags,
1122 timeout, &first);
1123 if (timeout < 0)
1124 return timeout;
1125 wait->first_signaled = first;
1126 } else {
1127 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1128 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1129 u64_to_user_ptr(timeline_wait->points),
1130 timeline_wait->count_handles,
1131 timeline_wait->flags,
1132 timeout, &first);
1133 if (timeout < 0)
1134 return timeout;
1135 timeline_wait->first_signaled = first;
1136 }
1137 return 0;
1138}
1139
1140static int drm_syncobj_array_find(struct drm_file *file_private,
1141 void __user *user_handles,
1142 uint32_t count_handles,
1143 struct drm_syncobj ***syncobjs_out)
1144{
1145 uint32_t i, *handles;
1146 struct drm_syncobj **syncobjs;
1147 int ret;
1148
1149 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1150 if (handles == NULL)
1151 return -ENOMEM;
1152
1153 if (copy_from_user(handles, user_handles,
1154 sizeof(uint32_t) * count_handles)) {
1155 ret = -EFAULT;
1156 goto err_free_handles;
1157 }
1158
1159 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1160 if (syncobjs == NULL) {
1161 ret = -ENOMEM;
1162 goto err_free_handles;
1163 }
1164
1165 for (i = 0; i < count_handles; i++) {
1166 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1167 if (!syncobjs[i]) {
1168 ret = -ENOENT;
1169 goto err_put_syncobjs;
1170 }
1171 }
1172
1173 kfree(handles);
1174 *syncobjs_out = syncobjs;
1175 return 0;
1176
1177err_put_syncobjs:
1178 while (i-- > 0)
1179 drm_syncobj_put(syncobjs[i]);
1180 kfree(syncobjs);
1181err_free_handles:
1182 kfree(handles);
1183
1184 return ret;
1185}
1186
1187static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1188 uint32_t count)
1189{
1190 uint32_t i;
1191
1192 for (i = 0; i < count; i++)
1193 drm_syncobj_put(syncobjs[i]);
1194 kfree(syncobjs);
1195}
1196
1197int
1198drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1199 struct drm_file *file_private)
1200{
1201 struct drm_syncobj_wait *args = data;
1202 struct drm_syncobj **syncobjs;
1203 int ret = 0;
1204
1205 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1206 return -EOPNOTSUPP;
1207
1208 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1209 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1210 return -EINVAL;
1211
1212 if (args->count_handles == 0)
1213 return -EINVAL;
1214
1215 ret = drm_syncobj_array_find(file_private,
1216 u64_to_user_ptr(args->handles),
1217 args->count_handles,
1218 &syncobjs);
1219 if (ret < 0)
1220 return ret;
1221
1222 ret = drm_syncobj_array_wait(dev, file_private,
1223 args, NULL, syncobjs, false);
1224
1225 drm_syncobj_array_free(syncobjs, args->count_handles);
1226
1227 return ret;
1228}
1229
1230int
1231drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1232 struct drm_file *file_private)
1233{
1234 struct drm_syncobj_timeline_wait *args = data;
1235 struct drm_syncobj **syncobjs;
1236 int ret = 0;
1237
1238 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1239 return -EOPNOTSUPP;
1240
1241 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1242 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1243 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1244 return -EINVAL;
1245
1246 if (args->count_handles == 0)
1247 return -EINVAL;
1248
1249 ret = drm_syncobj_array_find(file_private,
1250 u64_to_user_ptr(args->handles),
1251 args->count_handles,
1252 &syncobjs);
1253 if (ret < 0)
1254 return ret;
1255
1256 ret = drm_syncobj_array_wait(dev, file_private,
1257 NULL, args, syncobjs, true);
1258
1259 drm_syncobj_array_free(syncobjs, args->count_handles);
1260
1261 return ret;
1262}
1263
1264
1265int
1266drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1267 struct drm_file *file_private)
1268{
1269 struct drm_syncobj_array *args = data;
1270 struct drm_syncobj **syncobjs;
1271 uint32_t i;
1272 int ret;
1273
1274 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1275 return -EOPNOTSUPP;
1276
1277 if (args->pad != 0)
1278 return -EINVAL;
1279
1280 if (args->count_handles == 0)
1281 return -EINVAL;
1282
1283 ret = drm_syncobj_array_find(file_private,
1284 u64_to_user_ptr(args->handles),
1285 args->count_handles,
1286 &syncobjs);
1287 if (ret < 0)
1288 return ret;
1289
1290 for (i = 0; i < args->count_handles; i++)
1291 drm_syncobj_replace_fence(syncobjs[i], NULL);
1292
1293 drm_syncobj_array_free(syncobjs, args->count_handles);
1294
1295 return 0;
1296}
1297
1298int
1299drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1300 struct drm_file *file_private)
1301{
1302 struct drm_syncobj_array *args = data;
1303 struct drm_syncobj **syncobjs;
1304 uint32_t i;
1305 int ret;
1306
1307 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1308 return -EOPNOTSUPP;
1309
1310 if (args->pad != 0)
1311 return -EINVAL;
1312
1313 if (args->count_handles == 0)
1314 return -EINVAL;
1315
1316 ret = drm_syncobj_array_find(file_private,
1317 u64_to_user_ptr(args->handles),
1318 args->count_handles,
1319 &syncobjs);
1320 if (ret < 0)
1321 return ret;
1322
1323 for (i = 0; i < args->count_handles; i++)
1324 drm_syncobj_assign_null_handle(syncobjs[i]);
1325
1326 drm_syncobj_array_free(syncobjs, args->count_handles);
1327
1328 return ret;
1329}
1330
1331int
1332drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1333 struct drm_file *file_private)
1334{
1335 struct drm_syncobj_timeline_array *args = data;
1336 struct drm_syncobj **syncobjs;
1337 struct dma_fence_chain **chains;
1338 uint64_t *points;
1339 uint32_t i, j;
1340 int ret;
1341
1342 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1343 return -EOPNOTSUPP;
1344
1345 if (args->flags != 0)
1346 return -EINVAL;
1347
1348 if (args->count_handles == 0)
1349 return -EINVAL;
1350
1351 ret = drm_syncobj_array_find(file_private,
1352 u64_to_user_ptr(args->handles),
1353 args->count_handles,
1354 &syncobjs);
1355 if (ret < 0)
1356 return ret;
1357
1358 points = kmalloc_array(args->count_handles, sizeof(*points),
1359 GFP_KERNEL);
1360 if (!points) {
1361 ret = -ENOMEM;
1362 goto out;
1363 }
1364 if (!u64_to_user_ptr(args->points)) {
1365 memset(points, 0, args->count_handles * sizeof(uint64_t));
1366 } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1367 sizeof(uint64_t) * args->count_handles)) {
1368 ret = -EFAULT;
1369 goto err_points;
1370 }
1371
1372 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1373 if (!chains) {
1374 ret = -ENOMEM;
1375 goto err_points;
1376 }
1377 for (i = 0; i < args->count_handles; i++) {
1378 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1379 if (!chains[i]) {
1380 for (j = 0; j < i; j++)
1381 kfree(chains[j]);
1382 ret = -ENOMEM;
1383 goto err_chains;
1384 }
1385 }
1386
1387 for (i = 0; i < args->count_handles; i++) {
1388 struct dma_fence *fence = dma_fence_get_stub();
1389
1390 drm_syncobj_add_point(syncobjs[i], chains[i],
1391 fence, points[i]);
1392 dma_fence_put(fence);
1393 }
1394err_chains:
1395 kfree(chains);
1396err_points:
1397 kfree(points);
1398out:
1399 drm_syncobj_array_free(syncobjs, args->count_handles);
1400
1401 return ret;
1402}
1403
1404int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1405 struct drm_file *file_private)
1406{
1407 struct drm_syncobj_timeline_array *args = data;
1408 struct drm_syncobj **syncobjs;
1409 uint64_t __user *points = u64_to_user_ptr(args->points);
1410 uint32_t i;
1411 int ret;
1412
1413 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1414 return -EOPNOTSUPP;
1415
1416 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1417 return -EINVAL;
1418
1419 if (args->count_handles == 0)
1420 return -EINVAL;
1421
1422 ret = drm_syncobj_array_find(file_private,
1423 u64_to_user_ptr(args->handles),
1424 args->count_handles,
1425 &syncobjs);
1426 if (ret < 0)
1427 return ret;
1428
1429 for (i = 0; i < args->count_handles; i++) {
1430 struct dma_fence_chain *chain;
1431 struct dma_fence *fence;
1432 uint64_t point;
1433
1434 fence = drm_syncobj_fence_get(syncobjs[i]);
1435 chain = to_dma_fence_chain(fence);
1436 if (chain) {
1437 struct dma_fence *iter, *last_signaled =
1438 dma_fence_get(fence);
1439
1440 if (args->flags &
1441 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1442 point = fence->seqno;
1443 } else {
1444 dma_fence_chain_for_each(iter, fence) {
1445 if (iter->context != fence->context) {
1446 dma_fence_put(iter);
1447
1448
1449 break;
1450 }
1451 dma_fence_put(last_signaled);
1452 last_signaled = dma_fence_get(iter);
1453 }
1454 point = dma_fence_is_signaled(last_signaled) ?
1455 last_signaled->seqno :
1456 to_dma_fence_chain(last_signaled)->prev_seqno;
1457 }
1458 dma_fence_put(last_signaled);
1459 } else {
1460 point = 0;
1461 }
1462 dma_fence_put(fence);
1463 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1464 ret = ret ? -EFAULT : 0;
1465 if (ret)
1466 break;
1467 }
1468 drm_syncobj_array_free(syncobjs, args->count_handles);
1469
1470 return ret;
1471}
1472