1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49#include <linux/anon_inodes.h>
50#include <linux/file.h>
51#include <linux/fs.h>
52#include <linux/sched/signal.h>
53#include <linux/sync_file.h>
54#include <linux/uaccess.h>
55
56#include <drm/drm_drv.h>
57#include <drm/drm_file.h>
58#include <drm/drm_gem.h>
59#include <drm/drm_print.h>
60#include <drm/drm_syncobj.h>
61
62#include "drm_internal.h"
63
64struct syncobj_wait_entry {
65 struct list_head node;
66 struct task_struct *task;
67 struct dma_fence *fence;
68 struct dma_fence_cb fence_cb;
69 u64 point;
70};
71
72static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
73 struct syncobj_wait_entry *wait);
74
75
76
77
78
79
80
81
82
83struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
84 u32 handle)
85{
86 struct drm_syncobj *syncobj;
87
88 spin_lock(&file_private->syncobj_table_lock);
89
90
91 syncobj = idr_find(&file_private->syncobj_idr, handle);
92 if (syncobj)
93 drm_syncobj_get(syncobj);
94
95 spin_unlock(&file_private->syncobj_table_lock);
96
97 return syncobj;
98}
99EXPORT_SYMBOL(drm_syncobj_find);
100
101static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
102 struct syncobj_wait_entry *wait)
103{
104 struct dma_fence *fence;
105
106 if (wait->fence)
107 return;
108
109 spin_lock(&syncobj->lock);
110
111
112
113
114 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
115 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
116 dma_fence_put(fence);
117 list_add_tail(&wait->node, &syncobj->cb_list);
118 } else if (!fence) {
119 wait->fence = dma_fence_get_stub();
120 } else {
121 wait->fence = fence;
122 }
123 spin_unlock(&syncobj->lock);
124}
125
126static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
127 struct syncobj_wait_entry *wait)
128{
129 if (!wait->node.next)
130 return;
131
132 spin_lock(&syncobj->lock);
133 list_del_init(&wait->node);
134 spin_unlock(&syncobj->lock);
135}
136
137
138
139
140
141
142
143
144
145
146void drm_syncobj_add_point(struct drm_syncobj *syncobj,
147 struct dma_fence_chain *chain,
148 struct dma_fence *fence,
149 uint64_t point)
150{
151 struct syncobj_wait_entry *cur, *tmp;
152 struct dma_fence *prev;
153
154 dma_fence_get(fence);
155
156 spin_lock(&syncobj->lock);
157
158 prev = drm_syncobj_fence_get(syncobj);
159
160 if (prev && prev->seqno >= point)
161 DRM_ERROR("You are adding an unorder point to timeline!\n");
162 dma_fence_chain_init(chain, prev, fence, point);
163 rcu_assign_pointer(syncobj->fence, &chain->base);
164
165 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
166 syncobj_wait_syncobj_func(syncobj, cur);
167 spin_unlock(&syncobj->lock);
168
169
170 dma_fence_chain_for_each(fence, prev);
171 dma_fence_put(prev);
172}
173EXPORT_SYMBOL(drm_syncobj_add_point);
174
175
176
177
178
179
180
181
182void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
183 struct dma_fence *fence)
184{
185 struct dma_fence *old_fence;
186 struct syncobj_wait_entry *cur, *tmp;
187
188 if (fence)
189 dma_fence_get(fence);
190
191 spin_lock(&syncobj->lock);
192
193 old_fence = rcu_dereference_protected(syncobj->fence,
194 lockdep_is_held(&syncobj->lock));
195 rcu_assign_pointer(syncobj->fence, fence);
196
197 if (fence != old_fence) {
198 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
199 syncobj_wait_syncobj_func(syncobj, cur);
200 }
201
202 spin_unlock(&syncobj->lock);
203
204 dma_fence_put(old_fence);
205}
206EXPORT_SYMBOL(drm_syncobj_replace_fence);
207
208
209
210
211
212
213
214static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
215{
216 struct dma_fence *fence = dma_fence_get_stub();
217
218 drm_syncobj_replace_fence(syncobj, fence);
219 dma_fence_put(fence);
220}
221
222
223#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239int drm_syncobj_find_fence(struct drm_file *file_private,
240 u32 handle, u64 point, u64 flags,
241 struct dma_fence **fence)
242{
243 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
244 struct syncobj_wait_entry wait;
245 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
246 int ret;
247
248 if (!syncobj)
249 return -ENOENT;
250
251 *fence = drm_syncobj_fence_get(syncobj);
252 drm_syncobj_put(syncobj);
253
254 if (*fence) {
255 ret = dma_fence_chain_find_seqno(fence, point);
256 if (!ret)
257 return 0;
258 dma_fence_put(*fence);
259 } else {
260 ret = -EINVAL;
261 }
262
263 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
264 return ret;
265
266 memset(&wait, 0, sizeof(wait));
267 wait.task = current;
268 wait.point = point;
269 drm_syncobj_fence_add_wait(syncobj, &wait);
270
271 do {
272 set_current_state(TASK_INTERRUPTIBLE);
273 if (wait.fence) {
274 ret = 0;
275 break;
276 }
277 if (timeout == 0) {
278 ret = -ETIME;
279 break;
280 }
281
282 if (signal_pending(current)) {
283 ret = -ERESTARTSYS;
284 break;
285 }
286
287 timeout = schedule_timeout(timeout);
288 } while (1);
289
290 __set_current_state(TASK_RUNNING);
291 *fence = wait.fence;
292
293 if (wait.node.next)
294 drm_syncobj_remove_wait(syncobj, &wait);
295
296 return ret;
297}
298EXPORT_SYMBOL(drm_syncobj_find_fence);
299
300
301
302
303
304
305
306void drm_syncobj_free(struct kref *kref)
307{
308 struct drm_syncobj *syncobj = container_of(kref,
309 struct drm_syncobj,
310 refcount);
311 drm_syncobj_replace_fence(syncobj, NULL);
312 kfree(syncobj);
313}
314EXPORT_SYMBOL(drm_syncobj_free);
315
316
317
318
319
320
321
322
323
324
325
326
327
328int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
329 struct dma_fence *fence)
330{
331 struct drm_syncobj *syncobj;
332
333 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
334 if (!syncobj)
335 return -ENOMEM;
336
337 kref_init(&syncobj->refcount);
338 INIT_LIST_HEAD(&syncobj->cb_list);
339 spin_lock_init(&syncobj->lock);
340
341 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
342 drm_syncobj_assign_null_handle(syncobj);
343
344 if (fence)
345 drm_syncobj_replace_fence(syncobj, fence);
346
347 *out_syncobj = syncobj;
348 return 0;
349}
350EXPORT_SYMBOL(drm_syncobj_create);
351
352
353
354
355
356
357
358
359
360
361
362
363int drm_syncobj_get_handle(struct drm_file *file_private,
364 struct drm_syncobj *syncobj, u32 *handle)
365{
366 int ret;
367
368
369 drm_syncobj_get(syncobj);
370
371 idr_preload(GFP_KERNEL);
372 spin_lock(&file_private->syncobj_table_lock);
373 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
374 spin_unlock(&file_private->syncobj_table_lock);
375
376 idr_preload_end();
377
378 if (ret < 0) {
379 drm_syncobj_put(syncobj);
380 return ret;
381 }
382
383 *handle = ret;
384 return 0;
385}
386EXPORT_SYMBOL(drm_syncobj_get_handle);
387
388static int drm_syncobj_create_as_handle(struct drm_file *file_private,
389 u32 *handle, uint32_t flags)
390{
391 int ret;
392 struct drm_syncobj *syncobj;
393
394 ret = drm_syncobj_create(&syncobj, flags, NULL);
395 if (ret)
396 return ret;
397
398 ret = drm_syncobj_get_handle(file_private, syncobj, handle);
399 drm_syncobj_put(syncobj);
400 return ret;
401}
402
403static int drm_syncobj_destroy(struct drm_file *file_private,
404 u32 handle)
405{
406 struct drm_syncobj *syncobj;
407
408 spin_lock(&file_private->syncobj_table_lock);
409 syncobj = idr_remove(&file_private->syncobj_idr, handle);
410 spin_unlock(&file_private->syncobj_table_lock);
411
412 if (!syncobj)
413 return -EINVAL;
414
415 drm_syncobj_put(syncobj);
416 return 0;
417}
418
419static int drm_syncobj_file_release(struct inode *inode, struct file *file)
420{
421 struct drm_syncobj *syncobj = file->private_data;
422
423 drm_syncobj_put(syncobj);
424 return 0;
425}
426
427static const struct file_operations drm_syncobj_file_fops = {
428 .release = drm_syncobj_file_release,
429};
430
431
432
433
434
435
436
437
438
439
440int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
441{
442 struct file *file;
443 int fd;
444
445 fd = get_unused_fd_flags(O_CLOEXEC);
446 if (fd < 0)
447 return fd;
448
449 file = anon_inode_getfile("syncobj_file",
450 &drm_syncobj_file_fops,
451 syncobj, 0);
452 if (IS_ERR(file)) {
453 put_unused_fd(fd);
454 return PTR_ERR(file);
455 }
456
457 drm_syncobj_get(syncobj);
458 fd_install(fd, file);
459
460 *p_fd = fd;
461 return 0;
462}
463EXPORT_SYMBOL(drm_syncobj_get_fd);
464
465static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
466 u32 handle, int *p_fd)
467{
468 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
469 int ret;
470
471 if (!syncobj)
472 return -EINVAL;
473
474 ret = drm_syncobj_get_fd(syncobj, p_fd);
475 drm_syncobj_put(syncobj);
476 return ret;
477}
478
479static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
480 int fd, u32 *handle)
481{
482 struct drm_syncobj *syncobj;
483 struct fd f = fdget(fd);
484 int ret;
485
486 if (!f.file)
487 return -EINVAL;
488
489 if (f.file->f_op != &drm_syncobj_file_fops) {
490 fdput(f);
491 return -EINVAL;
492 }
493
494
495 syncobj = f.file->private_data;
496 drm_syncobj_get(syncobj);
497
498 idr_preload(GFP_KERNEL);
499 spin_lock(&file_private->syncobj_table_lock);
500 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
501 spin_unlock(&file_private->syncobj_table_lock);
502 idr_preload_end();
503
504 if (ret > 0) {
505 *handle = ret;
506 ret = 0;
507 } else
508 drm_syncobj_put(syncobj);
509
510 fdput(f);
511 return ret;
512}
513
514static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
515 int fd, int handle)
516{
517 struct dma_fence *fence = sync_file_get_fence(fd);
518 struct drm_syncobj *syncobj;
519
520 if (!fence)
521 return -EINVAL;
522
523 syncobj = drm_syncobj_find(file_private, handle);
524 if (!syncobj) {
525 dma_fence_put(fence);
526 return -ENOENT;
527 }
528
529 drm_syncobj_replace_fence(syncobj, fence);
530 dma_fence_put(fence);
531 drm_syncobj_put(syncobj);
532 return 0;
533}
534
535static int drm_syncobj_export_sync_file(struct drm_file *file_private,
536 int handle, int *p_fd)
537{
538 int ret;
539 struct dma_fence *fence;
540 struct sync_file *sync_file;
541 int fd = get_unused_fd_flags(O_CLOEXEC);
542
543 if (fd < 0)
544 return fd;
545
546 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
547 if (ret)
548 goto err_put_fd;
549
550 sync_file = sync_file_create(fence);
551
552 dma_fence_put(fence);
553
554 if (!sync_file) {
555 ret = -EINVAL;
556 goto err_put_fd;
557 }
558
559 fd_install(fd, sync_file->file);
560
561 *p_fd = fd;
562 return 0;
563err_put_fd:
564 put_unused_fd(fd);
565 return ret;
566}
567
568
569
570
571
572
573
574void
575drm_syncobj_open(struct drm_file *file_private)
576{
577 idr_init_base(&file_private->syncobj_idr, 1);
578 spin_lock_init(&file_private->syncobj_table_lock);
579}
580
581static int
582drm_syncobj_release_handle(int id, void *ptr, void *data)
583{
584 struct drm_syncobj *syncobj = ptr;
585
586 drm_syncobj_put(syncobj);
587 return 0;
588}
589
590
591
592
593
594
595
596
597
598void
599drm_syncobj_release(struct drm_file *file_private)
600{
601 idr_for_each(&file_private->syncobj_idr,
602 &drm_syncobj_release_handle, file_private);
603 idr_destroy(&file_private->syncobj_idr);
604}
605
606int
607drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
608 struct drm_file *file_private)
609{
610 struct drm_syncobj_create *args = data;
611
612 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
613 return -EOPNOTSUPP;
614
615
616 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
617 return -EINVAL;
618
619 return drm_syncobj_create_as_handle(file_private,
620 &args->handle, args->flags);
621}
622
623int
624drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
625 struct drm_file *file_private)
626{
627 struct drm_syncobj_destroy *args = data;
628
629 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
630 return -EOPNOTSUPP;
631
632
633 if (args->pad)
634 return -EINVAL;
635 return drm_syncobj_destroy(file_private, args->handle);
636}
637
638int
639drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
640 struct drm_file *file_private)
641{
642 struct drm_syncobj_handle *args = data;
643
644 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
645 return -EOPNOTSUPP;
646
647 if (args->pad)
648 return -EINVAL;
649
650 if (args->flags != 0 &&
651 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
652 return -EINVAL;
653
654 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
655 return drm_syncobj_export_sync_file(file_private, args->handle,
656 &args->fd);
657
658 return drm_syncobj_handle_to_fd(file_private, args->handle,
659 &args->fd);
660}
661
662int
663drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
664 struct drm_file *file_private)
665{
666 struct drm_syncobj_handle *args = data;
667
668 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
669 return -EOPNOTSUPP;
670
671 if (args->pad)
672 return -EINVAL;
673
674 if (args->flags != 0 &&
675 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
676 return -EINVAL;
677
678 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
679 return drm_syncobj_import_sync_file_fence(file_private,
680 args->fd,
681 args->handle);
682
683 return drm_syncobj_fd_to_handle(file_private, args->fd,
684 &args->handle);
685}
686
687static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
688 struct drm_syncobj_transfer *args)
689{
690 struct drm_syncobj *timeline_syncobj = NULL;
691 struct dma_fence *fence;
692 struct dma_fence_chain *chain;
693 int ret;
694
695 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
696 if (!timeline_syncobj) {
697 return -ENOENT;
698 }
699 ret = drm_syncobj_find_fence(file_private, args->src_handle,
700 args->src_point, args->flags,
701 &fence);
702 if (ret)
703 goto err;
704 chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
705 if (!chain) {
706 ret = -ENOMEM;
707 goto err1;
708 }
709 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
710err1:
711 dma_fence_put(fence);
712err:
713 drm_syncobj_put(timeline_syncobj);
714
715 return ret;
716}
717
718static int
719drm_syncobj_transfer_to_binary(struct drm_file *file_private,
720 struct drm_syncobj_transfer *args)
721{
722 struct drm_syncobj *binary_syncobj = NULL;
723 struct dma_fence *fence;
724 int ret;
725
726 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
727 if (!binary_syncobj)
728 return -ENOENT;
729 ret = drm_syncobj_find_fence(file_private, args->src_handle,
730 args->src_point, args->flags, &fence);
731 if (ret)
732 goto err;
733 drm_syncobj_replace_fence(binary_syncobj, fence);
734 dma_fence_put(fence);
735err:
736 drm_syncobj_put(binary_syncobj);
737
738 return ret;
739}
740int
741drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
742 struct drm_file *file_private)
743{
744 struct drm_syncobj_transfer *args = data;
745 int ret;
746
747 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
748 return -EOPNOTSUPP;
749
750 if (args->pad)
751 return -EINVAL;
752
753 if (args->dst_point)
754 ret = drm_syncobj_transfer_to_timeline(file_private, args);
755 else
756 ret = drm_syncobj_transfer_to_binary(file_private, args);
757
758 return ret;
759}
760
761static void syncobj_wait_fence_func(struct dma_fence *fence,
762 struct dma_fence_cb *cb)
763{
764 struct syncobj_wait_entry *wait =
765 container_of(cb, struct syncobj_wait_entry, fence_cb);
766
767 wake_up_process(wait->task);
768}
769
770static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
771 struct syncobj_wait_entry *wait)
772{
773 struct dma_fence *fence;
774
775
776 fence = rcu_dereference_protected(syncobj->fence,
777 lockdep_is_held(&syncobj->lock));
778 dma_fence_get(fence);
779 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
780 dma_fence_put(fence);
781 return;
782 } else if (!fence) {
783 wait->fence = dma_fence_get_stub();
784 } else {
785 wait->fence = fence;
786 }
787
788 wake_up_process(wait->task);
789 list_del_init(&wait->node);
790}
791
792static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
793 void __user *user_points,
794 uint32_t count,
795 uint32_t flags,
796 signed long timeout,
797 uint32_t *idx)
798{
799 struct syncobj_wait_entry *entries;
800 struct dma_fence *fence;
801 uint64_t *points;
802 uint32_t signaled_count, i;
803
804 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
805 if (points == NULL)
806 return -ENOMEM;
807
808 if (!user_points) {
809 memset(points, 0, count * sizeof(uint64_t));
810
811 } else if (copy_from_user(points, user_points,
812 sizeof(uint64_t) * count)) {
813 timeout = -EFAULT;
814 goto err_free_points;
815 }
816
817 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
818 if (!entries) {
819 timeout = -ENOMEM;
820 goto err_free_points;
821 }
822
823
824
825
826
827 signaled_count = 0;
828 for (i = 0; i < count; ++i) {
829 struct dma_fence *fence;
830
831 entries[i].task = current;
832 entries[i].point = points[i];
833 fence = drm_syncobj_fence_get(syncobjs[i]);
834 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
835 dma_fence_put(fence);
836 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
837 continue;
838 } else {
839 timeout = -EINVAL;
840 goto cleanup_entries;
841 }
842 }
843
844 if (fence)
845 entries[i].fence = fence;
846 else
847 entries[i].fence = dma_fence_get_stub();
848
849 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
850 dma_fence_is_signaled(entries[i].fence)) {
851 if (signaled_count == 0 && idx)
852 *idx = i;
853 signaled_count++;
854 }
855 }
856
857 if (signaled_count == count ||
858 (signaled_count > 0 &&
859 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
860 goto cleanup_entries;
861
862
863
864
865
866
867
868
869 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
870 for (i = 0; i < count; ++i)
871 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
872 }
873
874 do {
875 set_current_state(TASK_INTERRUPTIBLE);
876
877 signaled_count = 0;
878 for (i = 0; i < count; ++i) {
879 fence = entries[i].fence;
880 if (!fence)
881 continue;
882
883 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
884 dma_fence_is_signaled(fence) ||
885 (!entries[i].fence_cb.func &&
886 dma_fence_add_callback(fence,
887 &entries[i].fence_cb,
888 syncobj_wait_fence_func))) {
889
890 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
891 signaled_count++;
892 } else {
893 if (idx)
894 *idx = i;
895 goto done_waiting;
896 }
897 }
898 }
899
900 if (signaled_count == count)
901 goto done_waiting;
902
903 if (timeout == 0) {
904 timeout = -ETIME;
905 goto done_waiting;
906 }
907
908 if (signal_pending(current)) {
909 timeout = -ERESTARTSYS;
910 goto done_waiting;
911 }
912
913 timeout = schedule_timeout(timeout);
914 } while (1);
915
916done_waiting:
917 __set_current_state(TASK_RUNNING);
918
919cleanup_entries:
920 for (i = 0; i < count; ++i) {
921 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
922 if (entries[i].fence_cb.func)
923 dma_fence_remove_callback(entries[i].fence,
924 &entries[i].fence_cb);
925 dma_fence_put(entries[i].fence);
926 }
927 kfree(entries);
928
929err_free_points:
930 kfree(points);
931
932 return timeout;
933}
934
935
936
937
938
939
940
941
942signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
943{
944 ktime_t abs_timeout, now;
945 u64 timeout_ns, timeout_jiffies64;
946
947
948 if (timeout_nsec == 0)
949 return 0;
950
951 abs_timeout = ns_to_ktime(timeout_nsec);
952 now = ktime_get();
953
954 if (!ktime_after(abs_timeout, now))
955 return 0;
956
957 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
958
959 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
960
961 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
962 return MAX_SCHEDULE_TIMEOUT - 1;
963
964 return timeout_jiffies64 + 1;
965}
966EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
967
968static int drm_syncobj_array_wait(struct drm_device *dev,
969 struct drm_file *file_private,
970 struct drm_syncobj_wait *wait,
971 struct drm_syncobj_timeline_wait *timeline_wait,
972 struct drm_syncobj **syncobjs, bool timeline)
973{
974 signed long timeout = 0;
975 uint32_t first = ~0;
976
977 if (!timeline) {
978 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
979 timeout = drm_syncobj_array_wait_timeout(syncobjs,
980 NULL,
981 wait->count_handles,
982 wait->flags,
983 timeout, &first);
984 if (timeout < 0)
985 return timeout;
986 wait->first_signaled = first;
987 } else {
988 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
989 timeout = drm_syncobj_array_wait_timeout(syncobjs,
990 u64_to_user_ptr(timeline_wait->points),
991 timeline_wait->count_handles,
992 timeline_wait->flags,
993 timeout, &first);
994 if (timeout < 0)
995 return timeout;
996 timeline_wait->first_signaled = first;
997 }
998 return 0;
999}
1000
1001static int drm_syncobj_array_find(struct drm_file *file_private,
1002 void __user *user_handles,
1003 uint32_t count_handles,
1004 struct drm_syncobj ***syncobjs_out)
1005{
1006 uint32_t i, *handles;
1007 struct drm_syncobj **syncobjs;
1008 int ret;
1009
1010 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1011 if (handles == NULL)
1012 return -ENOMEM;
1013
1014 if (copy_from_user(handles, user_handles,
1015 sizeof(uint32_t) * count_handles)) {
1016 ret = -EFAULT;
1017 goto err_free_handles;
1018 }
1019
1020 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1021 if (syncobjs == NULL) {
1022 ret = -ENOMEM;
1023 goto err_free_handles;
1024 }
1025
1026 for (i = 0; i < count_handles; i++) {
1027 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1028 if (!syncobjs[i]) {
1029 ret = -ENOENT;
1030 goto err_put_syncobjs;
1031 }
1032 }
1033
1034 kfree(handles);
1035 *syncobjs_out = syncobjs;
1036 return 0;
1037
1038err_put_syncobjs:
1039 while (i-- > 0)
1040 drm_syncobj_put(syncobjs[i]);
1041 kfree(syncobjs);
1042err_free_handles:
1043 kfree(handles);
1044
1045 return ret;
1046}
1047
1048static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1049 uint32_t count)
1050{
1051 uint32_t i;
1052 for (i = 0; i < count; i++)
1053 drm_syncobj_put(syncobjs[i]);
1054 kfree(syncobjs);
1055}
1056
1057int
1058drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1059 struct drm_file *file_private)
1060{
1061 struct drm_syncobj_wait *args = data;
1062 struct drm_syncobj **syncobjs;
1063 int ret = 0;
1064
1065 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1066 return -EOPNOTSUPP;
1067
1068 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1069 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1070 return -EINVAL;
1071
1072 if (args->count_handles == 0)
1073 return -EINVAL;
1074
1075 ret = drm_syncobj_array_find(file_private,
1076 u64_to_user_ptr(args->handles),
1077 args->count_handles,
1078 &syncobjs);
1079 if (ret < 0)
1080 return ret;
1081
1082 ret = drm_syncobj_array_wait(dev, file_private,
1083 args, NULL, syncobjs, false);
1084
1085 drm_syncobj_array_free(syncobjs, args->count_handles);
1086
1087 return ret;
1088}
1089
1090int
1091drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1092 struct drm_file *file_private)
1093{
1094 struct drm_syncobj_timeline_wait *args = data;
1095 struct drm_syncobj **syncobjs;
1096 int ret = 0;
1097
1098 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1099 return -EOPNOTSUPP;
1100
1101 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1102 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1103 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1104 return -EINVAL;
1105
1106 if (args->count_handles == 0)
1107 return -EINVAL;
1108
1109 ret = drm_syncobj_array_find(file_private,
1110 u64_to_user_ptr(args->handles),
1111 args->count_handles,
1112 &syncobjs);
1113 if (ret < 0)
1114 return ret;
1115
1116 ret = drm_syncobj_array_wait(dev, file_private,
1117 NULL, args, syncobjs, true);
1118
1119 drm_syncobj_array_free(syncobjs, args->count_handles);
1120
1121 return ret;
1122}
1123
1124
1125int
1126drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1127 struct drm_file *file_private)
1128{
1129 struct drm_syncobj_array *args = data;
1130 struct drm_syncobj **syncobjs;
1131 uint32_t i;
1132 int ret;
1133
1134 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1135 return -EOPNOTSUPP;
1136
1137 if (args->pad != 0)
1138 return -EINVAL;
1139
1140 if (args->count_handles == 0)
1141 return -EINVAL;
1142
1143 ret = drm_syncobj_array_find(file_private,
1144 u64_to_user_ptr(args->handles),
1145 args->count_handles,
1146 &syncobjs);
1147 if (ret < 0)
1148 return ret;
1149
1150 for (i = 0; i < args->count_handles; i++)
1151 drm_syncobj_replace_fence(syncobjs[i], NULL);
1152
1153 drm_syncobj_array_free(syncobjs, args->count_handles);
1154
1155 return 0;
1156}
1157
1158int
1159drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1160 struct drm_file *file_private)
1161{
1162 struct drm_syncobj_array *args = data;
1163 struct drm_syncobj **syncobjs;
1164 uint32_t i;
1165 int ret;
1166
1167 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1168 return -EOPNOTSUPP;
1169
1170 if (args->pad != 0)
1171 return -EINVAL;
1172
1173 if (args->count_handles == 0)
1174 return -EINVAL;
1175
1176 ret = drm_syncobj_array_find(file_private,
1177 u64_to_user_ptr(args->handles),
1178 args->count_handles,
1179 &syncobjs);
1180 if (ret < 0)
1181 return ret;
1182
1183 for (i = 0; i < args->count_handles; i++)
1184 drm_syncobj_assign_null_handle(syncobjs[i]);
1185
1186 drm_syncobj_array_free(syncobjs, args->count_handles);
1187
1188 return ret;
1189}
1190
1191int
1192drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1193 struct drm_file *file_private)
1194{
1195 struct drm_syncobj_timeline_array *args = data;
1196 struct drm_syncobj **syncobjs;
1197 struct dma_fence_chain **chains;
1198 uint64_t *points;
1199 uint32_t i, j;
1200 int ret;
1201
1202 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1203 return -EOPNOTSUPP;
1204
1205 if (args->pad != 0)
1206 return -EINVAL;
1207
1208 if (args->count_handles == 0)
1209 return -EINVAL;
1210
1211 ret = drm_syncobj_array_find(file_private,
1212 u64_to_user_ptr(args->handles),
1213 args->count_handles,
1214 &syncobjs);
1215 if (ret < 0)
1216 return ret;
1217
1218 points = kmalloc_array(args->count_handles, sizeof(*points),
1219 GFP_KERNEL);
1220 if (!points) {
1221 ret = -ENOMEM;
1222 goto out;
1223 }
1224 if (!u64_to_user_ptr(args->points)) {
1225 memset(points, 0, args->count_handles * sizeof(uint64_t));
1226 } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1227 sizeof(uint64_t) * args->count_handles)) {
1228 ret = -EFAULT;
1229 goto err_points;
1230 }
1231
1232 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1233 if (!chains) {
1234 ret = -ENOMEM;
1235 goto err_points;
1236 }
1237 for (i = 0; i < args->count_handles; i++) {
1238 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1239 if (!chains[i]) {
1240 for (j = 0; j < i; j++)
1241 kfree(chains[j]);
1242 ret = -ENOMEM;
1243 goto err_chains;
1244 }
1245 }
1246
1247 for (i = 0; i < args->count_handles; i++) {
1248 struct dma_fence *fence = dma_fence_get_stub();
1249
1250 drm_syncobj_add_point(syncobjs[i], chains[i],
1251 fence, points[i]);
1252 dma_fence_put(fence);
1253 }
1254err_chains:
1255 kfree(chains);
1256err_points:
1257 kfree(points);
1258out:
1259 drm_syncobj_array_free(syncobjs, args->count_handles);
1260
1261 return ret;
1262}
1263
1264int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1265 struct drm_file *file_private)
1266{
1267 struct drm_syncobj_timeline_array *args = data;
1268 struct drm_syncobj **syncobjs;
1269 uint64_t __user *points = u64_to_user_ptr(args->points);
1270 uint32_t i;
1271 int ret;
1272
1273 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1274 return -EOPNOTSUPP;
1275
1276 if (args->pad != 0)
1277 return -EINVAL;
1278
1279 if (args->count_handles == 0)
1280 return -EINVAL;
1281
1282 ret = drm_syncobj_array_find(file_private,
1283 u64_to_user_ptr(args->handles),
1284 args->count_handles,
1285 &syncobjs);
1286 if (ret < 0)
1287 return ret;
1288
1289 for (i = 0; i < args->count_handles; i++) {
1290 struct dma_fence_chain *chain;
1291 struct dma_fence *fence;
1292 uint64_t point;
1293
1294 fence = drm_syncobj_fence_get(syncobjs[i]);
1295 chain = to_dma_fence_chain(fence);
1296 if (chain) {
1297 struct dma_fence *iter, *last_signaled = NULL;
1298
1299 dma_fence_chain_for_each(iter, fence) {
1300 if (!iter)
1301 break;
1302 dma_fence_put(last_signaled);
1303 last_signaled = dma_fence_get(iter);
1304 if (!to_dma_fence_chain(last_signaled)->prev_seqno)
1305
1306
1307 break;
1308 }
1309 point = dma_fence_is_signaled(last_signaled) ?
1310 last_signaled->seqno :
1311 to_dma_fence_chain(last_signaled)->prev_seqno;
1312 dma_fence_put(last_signaled);
1313 } else {
1314 point = 0;
1315 }
1316 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1317 ret = ret ? -EFAULT : 0;
1318 if (ret)
1319 break;
1320 }
1321 drm_syncobj_array_free(syncobjs, args->count_handles);
1322
1323 return ret;
1324}
1325