1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186#include <linux/anon_inodes.h>
187#include <linux/file.h>
188#include <linux/fs.h>
189#include <linux/sched/signal.h>
190#include <linux/sync_file.h>
191#include <linux/uaccess.h>
192
193#include <drm/drm.h>
194#include <drm/drm_drv.h>
195#include <drm/drm_file.h>
196#include <drm/drm_gem.h>
197#include <drm/drm_print.h>
198#include <drm/drm_syncobj.h>
199#include <drm/drm_utils.h>
200
201#include "drm_internal.h"
202
203struct syncobj_wait_entry {
204 struct list_head node;
205 struct task_struct *task;
206 struct dma_fence *fence;
207 struct dma_fence_cb fence_cb;
208 u64 point;
209};
210
211static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
212 struct syncobj_wait_entry *wait);
213
214
215
216
217
218
219
220
221
222struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
223 u32 handle)
224{
225 struct drm_syncobj *syncobj;
226
227 spin_lock(&file_private->syncobj_table_lock);
228
229
230 syncobj = idr_find(&file_private->syncobj_idr, handle);
231 if (syncobj)
232 drm_syncobj_get(syncobj);
233
234 spin_unlock(&file_private->syncobj_table_lock);
235
236 return syncobj;
237}
238EXPORT_SYMBOL(drm_syncobj_find);
239
240static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
241 struct syncobj_wait_entry *wait)
242{
243 struct dma_fence *fence;
244
245 if (wait->fence)
246 return;
247
248 spin_lock(&syncobj->lock);
249
250
251
252
253 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
254 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
255 dma_fence_put(fence);
256 list_add_tail(&wait->node, &syncobj->cb_list);
257 } else if (!fence) {
258 wait->fence = dma_fence_get_stub();
259 } else {
260 wait->fence = fence;
261 }
262 spin_unlock(&syncobj->lock);
263}
264
265static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
266 struct syncobj_wait_entry *wait)
267{
268 if (!wait->node.next)
269 return;
270
271 spin_lock(&syncobj->lock);
272 list_del_init(&wait->node);
273 spin_unlock(&syncobj->lock);
274}
275
276
277
278
279
280
281
282
283
284
285void drm_syncobj_add_point(struct drm_syncobj *syncobj,
286 struct dma_fence_chain *chain,
287 struct dma_fence *fence,
288 uint64_t point)
289{
290 struct syncobj_wait_entry *cur, *tmp;
291 struct dma_fence *prev;
292
293 dma_fence_get(fence);
294
295 spin_lock(&syncobj->lock);
296
297 prev = drm_syncobj_fence_get(syncobj);
298
299 if (prev && prev->seqno >= point)
300 DRM_DEBUG("You are adding an unorder point to timeline!\n");
301 dma_fence_chain_init(chain, prev, fence, point);
302 rcu_assign_pointer(syncobj->fence, &chain->base);
303
304 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
305 syncobj_wait_syncobj_func(syncobj, cur);
306 spin_unlock(&syncobj->lock);
307
308
309 dma_fence_chain_for_each(fence, prev);
310 dma_fence_put(prev);
311}
312EXPORT_SYMBOL(drm_syncobj_add_point);
313
314
315
316
317
318
319
320
321void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
322 struct dma_fence *fence)
323{
324 struct dma_fence *old_fence;
325 struct syncobj_wait_entry *cur, *tmp;
326
327 if (fence)
328 dma_fence_get(fence);
329
330 spin_lock(&syncobj->lock);
331
332 old_fence = rcu_dereference_protected(syncobj->fence,
333 lockdep_is_held(&syncobj->lock));
334 rcu_assign_pointer(syncobj->fence, fence);
335
336 if (fence != old_fence) {
337 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
338 syncobj_wait_syncobj_func(syncobj, cur);
339 }
340
341 spin_unlock(&syncobj->lock);
342
343 dma_fence_put(old_fence);
344}
345EXPORT_SYMBOL(drm_syncobj_replace_fence);
346
347
348
349
350
351
352
353static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
354{
355 struct dma_fence *fence = dma_fence_allocate_private_stub();
356
357 if (IS_ERR(fence))
358 return PTR_ERR(fence);
359
360 drm_syncobj_replace_fence(syncobj, fence);
361 dma_fence_put(fence);
362 return 0;
363}
364
365
366#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382int drm_syncobj_find_fence(struct drm_file *file_private,
383 u32 handle, u64 point, u64 flags,
384 struct dma_fence **fence)
385{
386 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
387 struct syncobj_wait_entry wait;
388 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
389 int ret;
390
391 if (!syncobj)
392 return -ENOENT;
393
394
395
396
397
398 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
399 might_sleep();
400 lockdep_assert_none_held_once();
401 }
402
403 *fence = drm_syncobj_fence_get(syncobj);
404
405 if (*fence) {
406 ret = dma_fence_chain_find_seqno(fence, point);
407 if (!ret)
408 goto out;
409 dma_fence_put(*fence);
410 } else {
411 ret = -EINVAL;
412 }
413
414 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
415 goto out;
416
417 memset(&wait, 0, sizeof(wait));
418 wait.task = current;
419 wait.point = point;
420 drm_syncobj_fence_add_wait(syncobj, &wait);
421
422 do {
423 set_current_state(TASK_INTERRUPTIBLE);
424 if (wait.fence) {
425 ret = 0;
426 break;
427 }
428 if (timeout == 0) {
429 ret = -ETIME;
430 break;
431 }
432
433 if (signal_pending(current)) {
434 ret = -ERESTARTSYS;
435 break;
436 }
437
438 timeout = schedule_timeout(timeout);
439 } while (1);
440
441 __set_current_state(TASK_RUNNING);
442 *fence = wait.fence;
443
444 if (wait.node.next)
445 drm_syncobj_remove_wait(syncobj, &wait);
446
447out:
448 drm_syncobj_put(syncobj);
449
450 return ret;
451}
452EXPORT_SYMBOL(drm_syncobj_find_fence);
453
454
455
456
457
458
459
460void drm_syncobj_free(struct kref *kref)
461{
462 struct drm_syncobj *syncobj = container_of(kref,
463 struct drm_syncobj,
464 refcount);
465 drm_syncobj_replace_fence(syncobj, NULL);
466 kfree(syncobj);
467}
468EXPORT_SYMBOL(drm_syncobj_free);
469
470
471
472
473
474
475
476
477
478
479
480
481
482int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
483 struct dma_fence *fence)
484{
485 int ret;
486 struct drm_syncobj *syncobj;
487
488 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
489 if (!syncobj)
490 return -ENOMEM;
491
492 kref_init(&syncobj->refcount);
493 INIT_LIST_HEAD(&syncobj->cb_list);
494 spin_lock_init(&syncobj->lock);
495
496 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
497 ret = drm_syncobj_assign_null_handle(syncobj);
498 if (ret < 0) {
499 drm_syncobj_put(syncobj);
500 return ret;
501 }
502 }
503
504 if (fence)
505 drm_syncobj_replace_fence(syncobj, fence);
506
507 *out_syncobj = syncobj;
508 return 0;
509}
510EXPORT_SYMBOL(drm_syncobj_create);
511
512
513
514
515
516
517
518
519
520
521
522
523int drm_syncobj_get_handle(struct drm_file *file_private,
524 struct drm_syncobj *syncobj, u32 *handle)
525{
526 int ret;
527
528
529 drm_syncobj_get(syncobj);
530
531 idr_preload(GFP_KERNEL);
532 spin_lock(&file_private->syncobj_table_lock);
533 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
534 spin_unlock(&file_private->syncobj_table_lock);
535
536 idr_preload_end();
537
538 if (ret < 0) {
539 drm_syncobj_put(syncobj);
540 return ret;
541 }
542
543 *handle = ret;
544 return 0;
545}
546EXPORT_SYMBOL(drm_syncobj_get_handle);
547
548static int drm_syncobj_create_as_handle(struct drm_file *file_private,
549 u32 *handle, uint32_t flags)
550{
551 int ret;
552 struct drm_syncobj *syncobj;
553
554 ret = drm_syncobj_create(&syncobj, flags, NULL);
555 if (ret)
556 return ret;
557
558 ret = drm_syncobj_get_handle(file_private, syncobj, handle);
559 drm_syncobj_put(syncobj);
560 return ret;
561}
562
563static int drm_syncobj_destroy(struct drm_file *file_private,
564 u32 handle)
565{
566 struct drm_syncobj *syncobj;
567
568 spin_lock(&file_private->syncobj_table_lock);
569 syncobj = idr_remove(&file_private->syncobj_idr, handle);
570 spin_unlock(&file_private->syncobj_table_lock);
571
572 if (!syncobj)
573 return -EINVAL;
574
575 drm_syncobj_put(syncobj);
576 return 0;
577}
578
579static int drm_syncobj_file_release(struct inode *inode, struct file *file)
580{
581 struct drm_syncobj *syncobj = file->private_data;
582
583 drm_syncobj_put(syncobj);
584 return 0;
585}
586
587static const struct file_operations drm_syncobj_file_fops = {
588 .release = drm_syncobj_file_release,
589};
590
591
592
593
594
595
596
597
598
599
600int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
601{
602 struct file *file;
603 int fd;
604
605 fd = get_unused_fd_flags(O_CLOEXEC);
606 if (fd < 0)
607 return fd;
608
609 file = anon_inode_getfile("syncobj_file",
610 &drm_syncobj_file_fops,
611 syncobj, 0);
612 if (IS_ERR(file)) {
613 put_unused_fd(fd);
614 return PTR_ERR(file);
615 }
616
617 drm_syncobj_get(syncobj);
618 fd_install(fd, file);
619
620 *p_fd = fd;
621 return 0;
622}
623EXPORT_SYMBOL(drm_syncobj_get_fd);
624
625static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
626 u32 handle, int *p_fd)
627{
628 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
629 int ret;
630
631 if (!syncobj)
632 return -EINVAL;
633
634 ret = drm_syncobj_get_fd(syncobj, p_fd);
635 drm_syncobj_put(syncobj);
636 return ret;
637}
638
639static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
640 int fd, u32 *handle)
641{
642 struct drm_syncobj *syncobj;
643 struct fd f = fdget(fd);
644 int ret;
645
646 if (!f.file)
647 return -EINVAL;
648
649 if (f.file->f_op != &drm_syncobj_file_fops) {
650 fdput(f);
651 return -EINVAL;
652 }
653
654
655 syncobj = f.file->private_data;
656 drm_syncobj_get(syncobj);
657
658 idr_preload(GFP_KERNEL);
659 spin_lock(&file_private->syncobj_table_lock);
660 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
661 spin_unlock(&file_private->syncobj_table_lock);
662 idr_preload_end();
663
664 if (ret > 0) {
665 *handle = ret;
666 ret = 0;
667 } else
668 drm_syncobj_put(syncobj);
669
670 fdput(f);
671 return ret;
672}
673
674static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
675 int fd, int handle)
676{
677 struct dma_fence *fence = sync_file_get_fence(fd);
678 struct drm_syncobj *syncobj;
679
680 if (!fence)
681 return -EINVAL;
682
683 syncobj = drm_syncobj_find(file_private, handle);
684 if (!syncobj) {
685 dma_fence_put(fence);
686 return -ENOENT;
687 }
688
689 drm_syncobj_replace_fence(syncobj, fence);
690 dma_fence_put(fence);
691 drm_syncobj_put(syncobj);
692 return 0;
693}
694
695static int drm_syncobj_export_sync_file(struct drm_file *file_private,
696 int handle, int *p_fd)
697{
698 int ret;
699 struct dma_fence *fence;
700 struct sync_file *sync_file;
701 int fd = get_unused_fd_flags(O_CLOEXEC);
702
703 if (fd < 0)
704 return fd;
705
706 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
707 if (ret)
708 goto err_put_fd;
709
710 sync_file = sync_file_create(fence);
711
712 dma_fence_put(fence);
713
714 if (!sync_file) {
715 ret = -EINVAL;
716 goto err_put_fd;
717 }
718
719 fd_install(fd, sync_file->file);
720
721 *p_fd = fd;
722 return 0;
723err_put_fd:
724 put_unused_fd(fd);
725 return ret;
726}
727
728
729
730
731
732
733
734void
735drm_syncobj_open(struct drm_file *file_private)
736{
737 idr_init_base(&file_private->syncobj_idr, 1);
738 spin_lock_init(&file_private->syncobj_table_lock);
739}
740
741static int
742drm_syncobj_release_handle(int id, void *ptr, void *data)
743{
744 struct drm_syncobj *syncobj = ptr;
745
746 drm_syncobj_put(syncobj);
747 return 0;
748}
749
750
751
752
753
754
755
756
757
758void
759drm_syncobj_release(struct drm_file *file_private)
760{
761 idr_for_each(&file_private->syncobj_idr,
762 &drm_syncobj_release_handle, file_private);
763 idr_destroy(&file_private->syncobj_idr);
764}
765
766int
767drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
768 struct drm_file *file_private)
769{
770 struct drm_syncobj_create *args = data;
771
772 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
773 return -EOPNOTSUPP;
774
775
776 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
777 return -EINVAL;
778
779 return drm_syncobj_create_as_handle(file_private,
780 &args->handle, args->flags);
781}
782
783int
784drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
785 struct drm_file *file_private)
786{
787 struct drm_syncobj_destroy *args = data;
788
789 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
790 return -EOPNOTSUPP;
791
792
793 if (args->pad)
794 return -EINVAL;
795 return drm_syncobj_destroy(file_private, args->handle);
796}
797
798int
799drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
800 struct drm_file *file_private)
801{
802 struct drm_syncobj_handle *args = data;
803
804 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
805 return -EOPNOTSUPP;
806
807 if (args->pad)
808 return -EINVAL;
809
810 if (args->flags != 0 &&
811 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
812 return -EINVAL;
813
814 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
815 return drm_syncobj_export_sync_file(file_private, args->handle,
816 &args->fd);
817
818 return drm_syncobj_handle_to_fd(file_private, args->handle,
819 &args->fd);
820}
821
822int
823drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
824 struct drm_file *file_private)
825{
826 struct drm_syncobj_handle *args = data;
827
828 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
829 return -EOPNOTSUPP;
830
831 if (args->pad)
832 return -EINVAL;
833
834 if (args->flags != 0 &&
835 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
836 return -EINVAL;
837
838 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
839 return drm_syncobj_import_sync_file_fence(file_private,
840 args->fd,
841 args->handle);
842
843 return drm_syncobj_fd_to_handle(file_private, args->fd,
844 &args->handle);
845}
846
847static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
848 struct drm_syncobj_transfer *args)
849{
850 struct drm_syncobj *timeline_syncobj = NULL;
851 struct dma_fence *fence;
852 struct dma_fence_chain *chain;
853 int ret;
854
855 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
856 if (!timeline_syncobj) {
857 return -ENOENT;
858 }
859 ret = drm_syncobj_find_fence(file_private, args->src_handle,
860 args->src_point, args->flags,
861 &fence);
862 if (ret)
863 goto err;
864 chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
865 if (!chain) {
866 ret = -ENOMEM;
867 goto err1;
868 }
869 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
870err1:
871 dma_fence_put(fence);
872err:
873 drm_syncobj_put(timeline_syncobj);
874
875 return ret;
876}
877
878static int
879drm_syncobj_transfer_to_binary(struct drm_file *file_private,
880 struct drm_syncobj_transfer *args)
881{
882 struct drm_syncobj *binary_syncobj = NULL;
883 struct dma_fence *fence;
884 int ret;
885
886 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
887 if (!binary_syncobj)
888 return -ENOENT;
889 ret = drm_syncobj_find_fence(file_private, args->src_handle,
890 args->src_point, args->flags, &fence);
891 if (ret)
892 goto err;
893 drm_syncobj_replace_fence(binary_syncobj, fence);
894 dma_fence_put(fence);
895err:
896 drm_syncobj_put(binary_syncobj);
897
898 return ret;
899}
900int
901drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
902 struct drm_file *file_private)
903{
904 struct drm_syncobj_transfer *args = data;
905 int ret;
906
907 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
908 return -EOPNOTSUPP;
909
910 if (args->pad)
911 return -EINVAL;
912
913 if (args->dst_point)
914 ret = drm_syncobj_transfer_to_timeline(file_private, args);
915 else
916 ret = drm_syncobj_transfer_to_binary(file_private, args);
917
918 return ret;
919}
920
921static void syncobj_wait_fence_func(struct dma_fence *fence,
922 struct dma_fence_cb *cb)
923{
924 struct syncobj_wait_entry *wait =
925 container_of(cb, struct syncobj_wait_entry, fence_cb);
926
927 wake_up_process(wait->task);
928}
929
930static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
931 struct syncobj_wait_entry *wait)
932{
933 struct dma_fence *fence;
934
935
936 fence = rcu_dereference_protected(syncobj->fence,
937 lockdep_is_held(&syncobj->lock));
938 dma_fence_get(fence);
939 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
940 dma_fence_put(fence);
941 return;
942 } else if (!fence) {
943 wait->fence = dma_fence_get_stub();
944 } else {
945 wait->fence = fence;
946 }
947
948 wake_up_process(wait->task);
949 list_del_init(&wait->node);
950}
951
952static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
953 void __user *user_points,
954 uint32_t count,
955 uint32_t flags,
956 signed long timeout,
957 uint32_t *idx)
958{
959 struct syncobj_wait_entry *entries;
960 struct dma_fence *fence;
961 uint64_t *points;
962 uint32_t signaled_count, i;
963
964 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
965 lockdep_assert_none_held_once();
966
967 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
968 if (points == NULL)
969 return -ENOMEM;
970
971 if (!user_points) {
972 memset(points, 0, count * sizeof(uint64_t));
973
974 } else if (copy_from_user(points, user_points,
975 sizeof(uint64_t) * count)) {
976 timeout = -EFAULT;
977 goto err_free_points;
978 }
979
980 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
981 if (!entries) {
982 timeout = -ENOMEM;
983 goto err_free_points;
984 }
985
986
987
988
989
990 signaled_count = 0;
991 for (i = 0; i < count; ++i) {
992 struct dma_fence *fence;
993
994 entries[i].task = current;
995 entries[i].point = points[i];
996 fence = drm_syncobj_fence_get(syncobjs[i]);
997 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
998 dma_fence_put(fence);
999 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1000 continue;
1001 } else {
1002 timeout = -EINVAL;
1003 goto cleanup_entries;
1004 }
1005 }
1006
1007 if (fence)
1008 entries[i].fence = fence;
1009 else
1010 entries[i].fence = dma_fence_get_stub();
1011
1012 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1013 dma_fence_is_signaled(entries[i].fence)) {
1014 if (signaled_count == 0 && idx)
1015 *idx = i;
1016 signaled_count++;
1017 }
1018 }
1019
1020 if (signaled_count == count ||
1021 (signaled_count > 0 &&
1022 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1023 goto cleanup_entries;
1024
1025
1026
1027
1028
1029
1030
1031
1032 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1033 for (i = 0; i < count; ++i)
1034 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1035 }
1036
1037 do {
1038 set_current_state(TASK_INTERRUPTIBLE);
1039
1040 signaled_count = 0;
1041 for (i = 0; i < count; ++i) {
1042 fence = entries[i].fence;
1043 if (!fence)
1044 continue;
1045
1046 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1047 dma_fence_is_signaled(fence) ||
1048 (!entries[i].fence_cb.func &&
1049 dma_fence_add_callback(fence,
1050 &entries[i].fence_cb,
1051 syncobj_wait_fence_func))) {
1052
1053 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1054 signaled_count++;
1055 } else {
1056 if (idx)
1057 *idx = i;
1058 goto done_waiting;
1059 }
1060 }
1061 }
1062
1063 if (signaled_count == count)
1064 goto done_waiting;
1065
1066 if (timeout == 0) {
1067 timeout = -ETIME;
1068 goto done_waiting;
1069 }
1070
1071 if (signal_pending(current)) {
1072 timeout = -ERESTARTSYS;
1073 goto done_waiting;
1074 }
1075
1076 timeout = schedule_timeout(timeout);
1077 } while (1);
1078
1079done_waiting:
1080 __set_current_state(TASK_RUNNING);
1081
1082cleanup_entries:
1083 for (i = 0; i < count; ++i) {
1084 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1085 if (entries[i].fence_cb.func)
1086 dma_fence_remove_callback(entries[i].fence,
1087 &entries[i].fence_cb);
1088 dma_fence_put(entries[i].fence);
1089 }
1090 kfree(entries);
1091
1092err_free_points:
1093 kfree(points);
1094
1095 return timeout;
1096}
1097
1098
1099
1100
1101
1102
1103
1104
1105signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1106{
1107 ktime_t abs_timeout, now;
1108 u64 timeout_ns, timeout_jiffies64;
1109
1110
1111 if (timeout_nsec == 0)
1112 return 0;
1113
1114 abs_timeout = ns_to_ktime(timeout_nsec);
1115 now = ktime_get();
1116
1117 if (!ktime_after(abs_timeout, now))
1118 return 0;
1119
1120 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1121
1122 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1123
1124 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1125 return MAX_SCHEDULE_TIMEOUT - 1;
1126
1127 return timeout_jiffies64 + 1;
1128}
1129EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1130
1131static int drm_syncobj_array_wait(struct drm_device *dev,
1132 struct drm_file *file_private,
1133 struct drm_syncobj_wait *wait,
1134 struct drm_syncobj_timeline_wait *timeline_wait,
1135 struct drm_syncobj **syncobjs, bool timeline)
1136{
1137 signed long timeout = 0;
1138 uint32_t first = ~0;
1139
1140 if (!timeline) {
1141 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1142 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1143 NULL,
1144 wait->count_handles,
1145 wait->flags,
1146 timeout, &first);
1147 if (timeout < 0)
1148 return timeout;
1149 wait->first_signaled = first;
1150 } else {
1151 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1152 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1153 u64_to_user_ptr(timeline_wait->points),
1154 timeline_wait->count_handles,
1155 timeline_wait->flags,
1156 timeout, &first);
1157 if (timeout < 0)
1158 return timeout;
1159 timeline_wait->first_signaled = first;
1160 }
1161 return 0;
1162}
1163
1164static int drm_syncobj_array_find(struct drm_file *file_private,
1165 void __user *user_handles,
1166 uint32_t count_handles,
1167 struct drm_syncobj ***syncobjs_out)
1168{
1169 uint32_t i, *handles;
1170 struct drm_syncobj **syncobjs;
1171 int ret;
1172
1173 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1174 if (handles == NULL)
1175 return -ENOMEM;
1176
1177 if (copy_from_user(handles, user_handles,
1178 sizeof(uint32_t) * count_handles)) {
1179 ret = -EFAULT;
1180 goto err_free_handles;
1181 }
1182
1183 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1184 if (syncobjs == NULL) {
1185 ret = -ENOMEM;
1186 goto err_free_handles;
1187 }
1188
1189 for (i = 0; i < count_handles; i++) {
1190 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1191 if (!syncobjs[i]) {
1192 ret = -ENOENT;
1193 goto err_put_syncobjs;
1194 }
1195 }
1196
1197 kfree(handles);
1198 *syncobjs_out = syncobjs;
1199 return 0;
1200
1201err_put_syncobjs:
1202 while (i-- > 0)
1203 drm_syncobj_put(syncobjs[i]);
1204 kfree(syncobjs);
1205err_free_handles:
1206 kfree(handles);
1207
1208 return ret;
1209}
1210
1211static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1212 uint32_t count)
1213{
1214 uint32_t i;
1215
1216 for (i = 0; i < count; i++)
1217 drm_syncobj_put(syncobjs[i]);
1218 kfree(syncobjs);
1219}
1220
1221int
1222drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1223 struct drm_file *file_private)
1224{
1225 struct drm_syncobj_wait *args = data;
1226 struct drm_syncobj **syncobjs;
1227 int ret = 0;
1228
1229 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1230 return -EOPNOTSUPP;
1231
1232 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1233 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1234 return -EINVAL;
1235
1236 if (args->count_handles == 0)
1237 return -EINVAL;
1238
1239 ret = drm_syncobj_array_find(file_private,
1240 u64_to_user_ptr(args->handles),
1241 args->count_handles,
1242 &syncobjs);
1243 if (ret < 0)
1244 return ret;
1245
1246 ret = drm_syncobj_array_wait(dev, file_private,
1247 args, NULL, syncobjs, false);
1248
1249 drm_syncobj_array_free(syncobjs, args->count_handles);
1250
1251 return ret;
1252}
1253
1254int
1255drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1256 struct drm_file *file_private)
1257{
1258 struct drm_syncobj_timeline_wait *args = data;
1259 struct drm_syncobj **syncobjs;
1260 int ret = 0;
1261
1262 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1263 return -EOPNOTSUPP;
1264
1265 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1266 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1267 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1268 return -EINVAL;
1269
1270 if (args->count_handles == 0)
1271 return -EINVAL;
1272
1273 ret = drm_syncobj_array_find(file_private,
1274 u64_to_user_ptr(args->handles),
1275 args->count_handles,
1276 &syncobjs);
1277 if (ret < 0)
1278 return ret;
1279
1280 ret = drm_syncobj_array_wait(dev, file_private,
1281 NULL, args, syncobjs, true);
1282
1283 drm_syncobj_array_free(syncobjs, args->count_handles);
1284
1285 return ret;
1286}
1287
1288
1289int
1290drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1291 struct drm_file *file_private)
1292{
1293 struct drm_syncobj_array *args = data;
1294 struct drm_syncobj **syncobjs;
1295 uint32_t i;
1296 int ret;
1297
1298 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1299 return -EOPNOTSUPP;
1300
1301 if (args->pad != 0)
1302 return -EINVAL;
1303
1304 if (args->count_handles == 0)
1305 return -EINVAL;
1306
1307 ret = drm_syncobj_array_find(file_private,
1308 u64_to_user_ptr(args->handles),
1309 args->count_handles,
1310 &syncobjs);
1311 if (ret < 0)
1312 return ret;
1313
1314 for (i = 0; i < args->count_handles; i++)
1315 drm_syncobj_replace_fence(syncobjs[i], NULL);
1316
1317 drm_syncobj_array_free(syncobjs, args->count_handles);
1318
1319 return 0;
1320}
1321
1322int
1323drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1324 struct drm_file *file_private)
1325{
1326 struct drm_syncobj_array *args = data;
1327 struct drm_syncobj **syncobjs;
1328 uint32_t i;
1329 int ret;
1330
1331 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1332 return -EOPNOTSUPP;
1333
1334 if (args->pad != 0)
1335 return -EINVAL;
1336
1337 if (args->count_handles == 0)
1338 return -EINVAL;
1339
1340 ret = drm_syncobj_array_find(file_private,
1341 u64_to_user_ptr(args->handles),
1342 args->count_handles,
1343 &syncobjs);
1344 if (ret < 0)
1345 return ret;
1346
1347 for (i = 0; i < args->count_handles; i++) {
1348 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1349 if (ret < 0)
1350 break;
1351 }
1352
1353 drm_syncobj_array_free(syncobjs, args->count_handles);
1354
1355 return ret;
1356}
1357
1358int
1359drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1360 struct drm_file *file_private)
1361{
1362 struct drm_syncobj_timeline_array *args = data;
1363 struct drm_syncobj **syncobjs;
1364 struct dma_fence_chain **chains;
1365 uint64_t *points;
1366 uint32_t i, j;
1367 int ret;
1368
1369 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1370 return -EOPNOTSUPP;
1371
1372 if (args->flags != 0)
1373 return -EINVAL;
1374
1375 if (args->count_handles == 0)
1376 return -EINVAL;
1377
1378 ret = drm_syncobj_array_find(file_private,
1379 u64_to_user_ptr(args->handles),
1380 args->count_handles,
1381 &syncobjs);
1382 if (ret < 0)
1383 return ret;
1384
1385 points = kmalloc_array(args->count_handles, sizeof(*points),
1386 GFP_KERNEL);
1387 if (!points) {
1388 ret = -ENOMEM;
1389 goto out;
1390 }
1391 if (!u64_to_user_ptr(args->points)) {
1392 memset(points, 0, args->count_handles * sizeof(uint64_t));
1393 } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1394 sizeof(uint64_t) * args->count_handles)) {
1395 ret = -EFAULT;
1396 goto err_points;
1397 }
1398
1399 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1400 if (!chains) {
1401 ret = -ENOMEM;
1402 goto err_points;
1403 }
1404 for (i = 0; i < args->count_handles; i++) {
1405 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1406 if (!chains[i]) {
1407 for (j = 0; j < i; j++)
1408 kfree(chains[j]);
1409 ret = -ENOMEM;
1410 goto err_chains;
1411 }
1412 }
1413
1414 for (i = 0; i < args->count_handles; i++) {
1415 struct dma_fence *fence = dma_fence_get_stub();
1416
1417 drm_syncobj_add_point(syncobjs[i], chains[i],
1418 fence, points[i]);
1419 dma_fence_put(fence);
1420 }
1421err_chains:
1422 kfree(chains);
1423err_points:
1424 kfree(points);
1425out:
1426 drm_syncobj_array_free(syncobjs, args->count_handles);
1427
1428 return ret;
1429}
1430
1431int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1432 struct drm_file *file_private)
1433{
1434 struct drm_syncobj_timeline_array *args = data;
1435 struct drm_syncobj **syncobjs;
1436 uint64_t __user *points = u64_to_user_ptr(args->points);
1437 uint32_t i;
1438 int ret;
1439
1440 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1441 return -EOPNOTSUPP;
1442
1443 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1444 return -EINVAL;
1445
1446 if (args->count_handles == 0)
1447 return -EINVAL;
1448
1449 ret = drm_syncobj_array_find(file_private,
1450 u64_to_user_ptr(args->handles),
1451 args->count_handles,
1452 &syncobjs);
1453 if (ret < 0)
1454 return ret;
1455
1456 for (i = 0; i < args->count_handles; i++) {
1457 struct dma_fence_chain *chain;
1458 struct dma_fence *fence;
1459 uint64_t point;
1460
1461 fence = drm_syncobj_fence_get(syncobjs[i]);
1462 chain = to_dma_fence_chain(fence);
1463 if (chain) {
1464 struct dma_fence *iter, *last_signaled =
1465 dma_fence_get(fence);
1466
1467 if (args->flags &
1468 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1469 point = fence->seqno;
1470 } else {
1471 dma_fence_chain_for_each(iter, fence) {
1472 if (iter->context != fence->context) {
1473 dma_fence_put(iter);
1474
1475
1476 break;
1477 }
1478 dma_fence_put(last_signaled);
1479 last_signaled = dma_fence_get(iter);
1480 }
1481 point = dma_fence_is_signaled(last_signaled) ?
1482 last_signaled->seqno :
1483 to_dma_fence_chain(last_signaled)->prev_seqno;
1484 }
1485 dma_fence_put(last_signaled);
1486 } else {
1487 point = 0;
1488 }
1489 dma_fence_put(fence);
1490 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1491 ret = ret ? -EFAULT : 0;
1492 if (ret)
1493 break;
1494 }
1495 drm_syncobj_array_free(syncobjs, args->count_handles);
1496
1497 return ret;
1498}
1499