1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <drm/drm_vblank.h>
28#include <drm/drmP.h>
29#include <linux/export.h>
30
31#include "drm_trace.h"
32#include "drm_internal.h"
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72#define DRM_TIMESTAMP_MAXRETRIES 3
73
74
75
76
77#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
78
79static bool
80drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
81 ktime_t *tvblank, bool in_vblank_irq);
82
83static unsigned int drm_timestamp_precision = 20;
84
85static int drm_vblank_offdelay = 5000;
86
87module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
88module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
89MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
90MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
91
92static void store_vblank(struct drm_device *dev, unsigned int pipe,
93 u32 vblank_count_inc,
94 ktime_t t_vblank, u32 last)
95{
96 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
97
98 assert_spin_locked(&dev->vblank_time_lock);
99
100 vblank->last = last;
101
102 write_seqlock(&vblank->seqlock);
103 vblank->time = t_vblank;
104 vblank->count += vblank_count_inc;
105 write_sequnlock(&vblank->seqlock);
106}
107
108
109
110
111
112static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
113{
114 WARN_ON_ONCE(dev->max_vblank_count != 0);
115 return 0;
116}
117
118static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
119{
120 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
121 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
122
123 if (crtc->funcs->get_vblank_counter)
124 return crtc->funcs->get_vblank_counter(crtc);
125 }
126
127 if (dev->driver->get_vblank_counter)
128 return dev->driver->get_vblank_counter(dev, pipe);
129
130 return drm_vblank_no_hw_counter(dev, pipe);
131}
132
133
134
135
136
137
138
139
140
141
142static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
143{
144 u32 cur_vblank;
145 bool rc;
146 ktime_t t_vblank;
147 int count = DRM_TIMESTAMP_MAXRETRIES;
148
149 spin_lock(&dev->vblank_time_lock);
150
151
152
153
154
155 do {
156 cur_vblank = __get_vblank_counter(dev, pipe);
157 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
158 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
159
160
161
162
163
164
165 if (!rc)
166 t_vblank = 0;
167
168
169
170
171
172 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
173
174 spin_unlock(&dev->vblank_time_lock);
175}
176
177
178
179
180
181
182
183
184
185
186
187
188
189static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
190 bool in_vblank_irq)
191{
192 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
193 u32 cur_vblank, diff;
194 bool rc;
195 ktime_t t_vblank;
196 int count = DRM_TIMESTAMP_MAXRETRIES;
197 int framedur_ns = vblank->framedur_ns;
198
199
200
201
202
203
204
205
206
207
208
209
210
211 do {
212 cur_vblank = __get_vblank_counter(dev, pipe);
213 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
214 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
215
216 if (dev->max_vblank_count != 0) {
217
218 diff = (cur_vblank - vblank->last) & dev->max_vblank_count;
219 } else if (rc && framedur_ns) {
220 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
221
222
223
224
225
226
227 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
228
229 if (diff == 0 && in_vblank_irq)
230 DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored."
231 " diff_ns = %lld, framedur_ns = %d)\n",
232 pipe, (long long) diff_ns, framedur_ns);
233 } else {
234
235 diff = in_vblank_irq ? 1 : 0;
236 }
237
238
239
240
241
242
243
244
245
246
247 if (diff > 1 && (vblank->inmodeset & 0x2)) {
248 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
249 " due to pre-modeset.\n", pipe, diff);
250 diff = 1;
251 }
252
253 DRM_DEBUG_VBL("updating vblank count on crtc %u:"
254 " current=%llu, diff=%u, hw=%u hw_last=%u\n",
255 pipe, vblank->count, diff, cur_vblank, vblank->last);
256
257 if (diff == 0) {
258 WARN_ON_ONCE(cur_vblank != vblank->last);
259 return;
260 }
261
262
263
264
265
266
267
268 if (!rc && !in_vblank_irq)
269 t_vblank = 0;
270
271 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
272}
273
274static u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
275{
276 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
277
278 if (WARN_ON(pipe >= dev->num_crtcs))
279 return 0;
280
281 return vblank->count;
282}
283
284
285
286
287
288
289
290
291
292
293
294
295u32 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
296{
297 struct drm_device *dev = crtc->dev;
298 unsigned int pipe = drm_crtc_index(crtc);
299 u32 vblank;
300 unsigned long flags;
301
302 WARN_ONCE(drm_debug & DRM_UT_VBL && !dev->driver->get_vblank_timestamp,
303 "This function requires support for accurate vblank timestamps.");
304
305 spin_lock_irqsave(&dev->vblank_time_lock, flags);
306
307 drm_update_vblank_count(dev, pipe, false);
308 vblank = drm_vblank_count(dev, pipe);
309
310 spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
311
312 return vblank;
313}
314EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
315
316static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
317{
318 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
319 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
320
321 if (crtc->funcs->disable_vblank) {
322 crtc->funcs->disable_vblank(crtc);
323 return;
324 }
325 }
326
327 dev->driver->disable_vblank(dev, pipe);
328}
329
330
331
332
333
334
335
336void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
337{
338 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
339 unsigned long irqflags;
340
341 assert_spin_locked(&dev->vbl_lock);
342
343
344
345
346
347 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
348
349
350
351
352
353
354 if (vblank->enabled) {
355 __disable_vblank(dev, pipe);
356 vblank->enabled = false;
357 }
358
359
360
361
362
363
364
365 drm_update_vblank_count(dev, pipe, false);
366
367 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
368}
369
370static void vblank_disable_fn(struct timer_list *t)
371{
372 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
373 struct drm_device *dev = vblank->dev;
374 unsigned int pipe = vblank->pipe;
375 unsigned long irqflags;
376
377 spin_lock_irqsave(&dev->vbl_lock, irqflags);
378 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
379 DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
380 drm_vblank_disable_and_save(dev, pipe);
381 }
382 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
383}
384
385void drm_vblank_cleanup(struct drm_device *dev)
386{
387 unsigned int pipe;
388
389
390 if (dev->num_crtcs == 0)
391 return;
392
393 for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
394 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
395
396 WARN_ON(READ_ONCE(vblank->enabled) &&
397 drm_core_check_feature(dev, DRIVER_MODESET));
398
399 del_timer_sync(&vblank->disable_timer);
400 }
401
402 kfree(dev->vblank);
403
404 dev->num_crtcs = 0;
405}
406
407
408
409
410
411
412
413
414
415
416
417
418
419int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
420{
421 int ret = -ENOMEM;
422 unsigned int i;
423
424 spin_lock_init(&dev->vbl_lock);
425 spin_lock_init(&dev->vblank_time_lock);
426
427 dev->num_crtcs = num_crtcs;
428
429 dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
430 if (!dev->vblank)
431 goto err;
432
433 for (i = 0; i < num_crtcs; i++) {
434 struct drm_vblank_crtc *vblank = &dev->vblank[i];
435
436 vblank->dev = dev;
437 vblank->pipe = i;
438 init_waitqueue_head(&vblank->queue);
439 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
440 seqlock_init(&vblank->seqlock);
441 }
442
443 DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
444
445
446 if (dev->driver->get_vblank_timestamp)
447 DRM_INFO("Driver supports precise vblank timestamp query.\n");
448 else
449 DRM_INFO("No driver support for vblank timestamp query.\n");
450
451
452 if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
453 dev->vblank_disable_immediate = false;
454 DRM_INFO("Setting vblank_disable_immediate to false because "
455 "get_vblank_timestamp == NULL\n");
456 }
457
458 return 0;
459
460err:
461 dev->num_crtcs = 0;
462 return ret;
463}
464EXPORT_SYMBOL(drm_vblank_init);
465
466
467
468
469
470
471
472
473
474wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
475{
476 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
477}
478EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
479
480
481
482
483
484
485
486
487
488
489
490
491
492void drm_calc_timestamping_constants(struct drm_crtc *crtc,
493 const struct drm_display_mode *mode)
494{
495 struct drm_device *dev = crtc->dev;
496 unsigned int pipe = drm_crtc_index(crtc);
497 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
498 int linedur_ns = 0, framedur_ns = 0;
499 int dotclock = mode->crtc_clock;
500
501 if (!dev->num_crtcs)
502 return;
503
504 if (WARN_ON(pipe >= dev->num_crtcs))
505 return;
506
507
508 if (dotclock > 0) {
509 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
510
511
512
513
514
515
516 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
517 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
518
519
520
521
522 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
523 framedur_ns /= 2;
524 } else
525 DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
526 crtc->base.id);
527
528 vblank->linedur_ns = linedur_ns;
529 vblank->framedur_ns = framedur_ns;
530 vblank->hwmode = *mode;
531
532 DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
533 crtc->base.id, mode->crtc_htotal,
534 mode->crtc_vtotal, mode->crtc_vdisplay);
535 DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
536 crtc->base.id, dotclock, framedur_ns, linedur_ns);
537}
538EXPORT_SYMBOL(drm_calc_timestamping_constants);
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
572 unsigned int pipe,
573 int *max_error,
574 ktime_t *vblank_time,
575 bool in_vblank_irq)
576{
577 struct timespec64 ts_etime, ts_vblank_time;
578 ktime_t stime, etime;
579 bool vbl_status;
580 struct drm_crtc *crtc;
581 const struct drm_display_mode *mode;
582 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
583 int vpos, hpos, i;
584 int delta_ns, duration_ns;
585
586 if (!drm_core_check_feature(dev, DRIVER_MODESET))
587 return false;
588
589 crtc = drm_crtc_from_index(dev, pipe);
590
591 if (pipe >= dev->num_crtcs || !crtc) {
592 DRM_ERROR("Invalid crtc %u\n", pipe);
593 return false;
594 }
595
596
597 if (!dev->driver->get_scanout_position) {
598 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
599 return false;
600 }
601
602 if (drm_drv_uses_atomic_modeset(dev))
603 mode = &vblank->hwmode;
604 else
605 mode = &crtc->hwmode;
606
607
608
609
610 if (mode->crtc_clock == 0) {
611 DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
612 WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
613
614 return false;
615 }
616
617
618
619
620
621
622
623
624 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
625
626
627
628
629 vbl_status = dev->driver->get_scanout_position(dev, pipe,
630 in_vblank_irq,
631 &vpos, &hpos,
632 &stime, &etime,
633 mode);
634
635
636 if (!vbl_status) {
637 DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
638 pipe);
639 return false;
640 }
641
642
643 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
644
645
646 if (duration_ns <= *max_error)
647 break;
648 }
649
650
651 if (i == DRM_TIMESTAMP_MAXRETRIES) {
652 DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
653 pipe, duration_ns/1000, *max_error/1000, i);
654 }
655
656
657 *max_error = duration_ns;
658
659
660
661
662
663 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
664 mode->crtc_clock);
665
666
667
668
669 *vblank_time = ktime_sub_ns(etime, delta_ns);
670
671 if ((drm_debug & DRM_UT_VBL) == 0)
672 return true;
673
674 ts_etime = ktime_to_timespec64(etime);
675 ts_vblank_time = ktime_to_timespec64(*vblank_time);
676
677 DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
678 pipe, hpos, vpos,
679 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
680 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
681 duration_ns / 1000, i);
682
683 return true;
684}
685EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708static bool
709drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
710 ktime_t *tvblank, bool in_vblank_irq)
711{
712 bool ret = false;
713
714
715 int max_error = (int) drm_timestamp_precision * 1000;
716
717
718 if (dev->driver->get_vblank_timestamp && (max_error > 0))
719 ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
720 tvblank, in_vblank_irq);
721
722
723
724
725 if (!ret)
726 *tvblank = ktime_get();
727
728 return ret;
729}
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
745{
746 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
747}
748EXPORT_SYMBOL(drm_crtc_vblank_count);
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
765 ktime_t *vblanktime)
766{
767 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
768 u64 vblank_count;
769 unsigned int seq;
770
771 if (WARN_ON(pipe >= dev->num_crtcs)) {
772 *vblanktime = 0;
773 return 0;
774 }
775
776 do {
777 seq = read_seqbegin(&vblank->seqlock);
778 vblank_count = vblank->count;
779 *vblanktime = vblank->time;
780 } while (read_seqretry(&vblank->seqlock, seq));
781
782 return vblank_count;
783}
784
785
786
787
788
789
790
791
792
793
794
795
796u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
797 ktime_t *vblanktime)
798{
799 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
800 vblanktime);
801}
802EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
803
804static void send_vblank_event(struct drm_device *dev,
805 struct drm_pending_vblank_event *e,
806 u64 seq, ktime_t now)
807{
808 struct timespec64 tv;
809
810 switch (e->event.base.type) {
811 case DRM_EVENT_VBLANK:
812 case DRM_EVENT_FLIP_COMPLETE:
813 tv = ktime_to_timespec64(now);
814 e->event.vbl.sequence = seq;
815
816
817
818
819
820 e->event.vbl.tv_sec = tv.tv_sec;
821 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
822 break;
823 case DRM_EVENT_CRTC_SEQUENCE:
824 if (seq)
825 e->event.seq.sequence = seq;
826 e->event.seq.time_ns = ktime_to_ns(now);
827 break;
828 }
829 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
830 drm_send_event_locked(dev, &e->base);
831}
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
872 struct drm_pending_vblank_event *e)
873{
874 struct drm_device *dev = crtc->dev;
875 unsigned int pipe = drm_crtc_index(crtc);
876
877 assert_spin_locked(&dev->event_lock);
878
879 e->pipe = pipe;
880 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
881 list_add_tail(&e->base.link, &dev->vblank_event_list);
882}
883EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
884
885
886
887
888
889
890
891
892
893
894
895
896void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
897 struct drm_pending_vblank_event *e)
898{
899 struct drm_device *dev = crtc->dev;
900 u64 seq;
901 unsigned int pipe = drm_crtc_index(crtc);
902 ktime_t now;
903
904 if (dev->num_crtcs > 0) {
905 seq = drm_vblank_count_and_time(dev, pipe, &now);
906 } else {
907 seq = 0;
908
909 now = ktime_get();
910 }
911 e->pipe = pipe;
912 send_vblank_event(dev, e, seq, now);
913}
914EXPORT_SYMBOL(drm_crtc_send_vblank_event);
915
916static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
917{
918 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
919 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
920
921 if (crtc->funcs->enable_vblank)
922 return crtc->funcs->enable_vblank(crtc);
923 }
924
925 return dev->driver->enable_vblank(dev, pipe);
926}
927
928static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
929{
930 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
931 int ret = 0;
932
933 assert_spin_locked(&dev->vbl_lock);
934
935 spin_lock(&dev->vblank_time_lock);
936
937 if (!vblank->enabled) {
938
939
940
941
942
943
944
945 ret = __enable_vblank(dev, pipe);
946 DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
947 if (ret) {
948 atomic_dec(&vblank->refcount);
949 } else {
950 drm_update_vblank_count(dev, pipe, 0);
951
952
953
954
955
956 WRITE_ONCE(vblank->enabled, true);
957 }
958 }
959
960 spin_unlock(&dev->vblank_time_lock);
961
962 return ret;
963}
964
965static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
966{
967 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
968 unsigned long irqflags;
969 int ret = 0;
970
971 if (!dev->num_crtcs)
972 return -EINVAL;
973
974 if (WARN_ON(pipe >= dev->num_crtcs))
975 return -EINVAL;
976
977 spin_lock_irqsave(&dev->vbl_lock, irqflags);
978
979 if (atomic_add_return(1, &vblank->refcount) == 1) {
980 ret = drm_vblank_enable(dev, pipe);
981 } else {
982 if (!vblank->enabled) {
983 atomic_dec(&vblank->refcount);
984 ret = -EINVAL;
985 }
986 }
987 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
988
989 return ret;
990}
991
992
993
994
995
996
997
998
999
1000
1001
1002int drm_crtc_vblank_get(struct drm_crtc *crtc)
1003{
1004 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1005}
1006EXPORT_SYMBOL(drm_crtc_vblank_get);
1007
1008static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1009{
1010 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1011
1012 if (WARN_ON(pipe >= dev->num_crtcs))
1013 return;
1014
1015 if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1016 return;
1017
1018
1019 if (atomic_dec_and_test(&vblank->refcount)) {
1020 if (drm_vblank_offdelay == 0)
1021 return;
1022 else if (drm_vblank_offdelay < 0)
1023 vblank_disable_fn(&vblank->disable_timer);
1024 else if (!dev->vblank_disable_immediate)
1025 mod_timer(&vblank->disable_timer,
1026 jiffies + ((drm_vblank_offdelay * HZ)/1000));
1027 }
1028}
1029
1030
1031
1032
1033
1034
1035
1036
1037void drm_crtc_vblank_put(struct drm_crtc *crtc)
1038{
1039 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1040}
1041EXPORT_SYMBOL(drm_crtc_vblank_put);
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1055{
1056 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1057 int ret;
1058 u32 last;
1059
1060 if (WARN_ON(pipe >= dev->num_crtcs))
1061 return;
1062
1063 ret = drm_vblank_get(dev, pipe);
1064 if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1065 return;
1066
1067 last = drm_vblank_count(dev, pipe);
1068
1069 ret = wait_event_timeout(vblank->queue,
1070 last != drm_vblank_count(dev, pipe),
1071 msecs_to_jiffies(100));
1072
1073 WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1074
1075 drm_vblank_put(dev, pipe);
1076}
1077EXPORT_SYMBOL(drm_wait_one_vblank);
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1088{
1089 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1090}
1091EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104void drm_crtc_vblank_off(struct drm_crtc *crtc)
1105{
1106 struct drm_device *dev = crtc->dev;
1107 unsigned int pipe = drm_crtc_index(crtc);
1108 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1109 struct drm_pending_vblank_event *e, *t;
1110
1111 ktime_t now;
1112 unsigned long irqflags;
1113 u64 seq;
1114
1115 if (WARN_ON(pipe >= dev->num_crtcs))
1116 return;
1117
1118 spin_lock_irqsave(&dev->event_lock, irqflags);
1119
1120 spin_lock(&dev->vbl_lock);
1121 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1122 pipe, vblank->enabled, vblank->inmodeset);
1123
1124
1125
1126 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1127 drm_vblank_disable_and_save(dev, pipe);
1128
1129 wake_up(&vblank->queue);
1130
1131
1132
1133
1134
1135 if (!vblank->inmodeset) {
1136 atomic_inc(&vblank->refcount);
1137 vblank->inmodeset = 1;
1138 }
1139 spin_unlock(&dev->vbl_lock);
1140
1141
1142 seq = drm_vblank_count_and_time(dev, pipe, &now);
1143
1144 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1145 if (e->pipe != pipe)
1146 continue;
1147 DRM_DEBUG("Sending premature vblank event on disable: "
1148 "wanted %llu, current %llu\n",
1149 e->sequence, seq);
1150 list_del(&e->base.link);
1151 drm_vblank_put(dev, pipe);
1152 send_vblank_event(dev, e, seq, now);
1153 }
1154 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1155
1156
1157
1158 vblank->hwmode.crtc_clock = 0;
1159}
1160EXPORT_SYMBOL(drm_crtc_vblank_off);
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1175{
1176 struct drm_device *dev = crtc->dev;
1177 unsigned long irqflags;
1178 unsigned int pipe = drm_crtc_index(crtc);
1179 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1180
1181 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1182
1183
1184
1185
1186 if (!vblank->inmodeset) {
1187 atomic_inc(&vblank->refcount);
1188 vblank->inmodeset = 1;
1189 }
1190 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1191
1192 WARN_ON(!list_empty(&dev->vblank_event_list));
1193}
1194EXPORT_SYMBOL(drm_crtc_vblank_reset);
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206void drm_crtc_vblank_on(struct drm_crtc *crtc)
1207{
1208 struct drm_device *dev = crtc->dev;
1209 unsigned int pipe = drm_crtc_index(crtc);
1210 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1211 unsigned long irqflags;
1212
1213 if (WARN_ON(pipe >= dev->num_crtcs))
1214 return;
1215
1216 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1217 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1218 pipe, vblank->enabled, vblank->inmodeset);
1219
1220
1221 if (vblank->inmodeset) {
1222 atomic_dec(&vblank->refcount);
1223 vblank->inmodeset = 0;
1224 }
1225
1226 drm_reset_vblank_timestamp(dev, pipe);
1227
1228
1229
1230
1231
1232 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1233 WARN_ON(drm_vblank_enable(dev, pipe));
1234 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1235}
1236EXPORT_SYMBOL(drm_crtc_vblank_on);
1237
1238static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1239 unsigned int pipe)
1240{
1241 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1242
1243
1244 if (!dev->num_crtcs)
1245 return;
1246
1247 if (WARN_ON(pipe >= dev->num_crtcs))
1248 return;
1249
1250
1251
1252
1253
1254
1255
1256
1257 if (!vblank->inmodeset) {
1258 vblank->inmodeset = 0x1;
1259 if (drm_vblank_get(dev, pipe) == 0)
1260 vblank->inmodeset |= 0x2;
1261 }
1262}
1263
1264static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1265 unsigned int pipe)
1266{
1267 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1268 unsigned long irqflags;
1269
1270
1271 if (!dev->num_crtcs)
1272 return;
1273
1274 if (WARN_ON(pipe >= dev->num_crtcs))
1275 return;
1276
1277 if (vblank->inmodeset) {
1278 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1279 drm_reset_vblank_timestamp(dev, pipe);
1280 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1281
1282 if (vblank->inmodeset & 0x2)
1283 drm_vblank_put(dev, pipe);
1284
1285 vblank->inmodeset = 0;
1286 }
1287}
1288
1289int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1290 struct drm_file *file_priv)
1291{
1292 struct drm_modeset_ctl *modeset = data;
1293 unsigned int pipe;
1294
1295
1296 if (!dev->num_crtcs)
1297 return 0;
1298
1299
1300 if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1301 return 0;
1302
1303 pipe = modeset->crtc;
1304 if (pipe >= dev->num_crtcs)
1305 return -EINVAL;
1306
1307 switch (modeset->cmd) {
1308 case _DRM_PRE_MODESET:
1309 drm_legacy_vblank_pre_modeset(dev, pipe);
1310 break;
1311 case _DRM_POST_MODESET:
1312 drm_legacy_vblank_post_modeset(dev, pipe);
1313 break;
1314 default:
1315 return -EINVAL;
1316 }
1317
1318 return 0;
1319}
1320
1321static inline bool vblank_passed(u64 seq, u64 ref)
1322{
1323 return (seq - ref) <= (1 << 23);
1324}
1325
1326static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1327 u64 req_seq,
1328 union drm_wait_vblank *vblwait,
1329 struct drm_file *file_priv)
1330{
1331 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1332 struct drm_pending_vblank_event *e;
1333 ktime_t now;
1334 unsigned long flags;
1335 u64 seq;
1336 int ret;
1337
1338 e = kzalloc(sizeof(*e), GFP_KERNEL);
1339 if (e == NULL) {
1340 ret = -ENOMEM;
1341 goto err_put;
1342 }
1343
1344 e->pipe = pipe;
1345 e->event.base.type = DRM_EVENT_VBLANK;
1346 e->event.base.length = sizeof(e->event.vbl);
1347 e->event.vbl.user_data = vblwait->request.signal;
1348 e->event.vbl.crtc_id = 0;
1349 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1350 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1351 if (crtc)
1352 e->event.vbl.crtc_id = crtc->base.id;
1353 }
1354
1355 spin_lock_irqsave(&dev->event_lock, flags);
1356
1357
1358
1359
1360
1361
1362
1363 if (!READ_ONCE(vblank->enabled)) {
1364 ret = -EINVAL;
1365 goto err_unlock;
1366 }
1367
1368 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1369 &e->event.base);
1370
1371 if (ret)
1372 goto err_unlock;
1373
1374 seq = drm_vblank_count_and_time(dev, pipe, &now);
1375
1376 DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1377 req_seq, seq, pipe);
1378
1379 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1380
1381 e->sequence = req_seq;
1382 if (vblank_passed(seq, req_seq)) {
1383 drm_vblank_put(dev, pipe);
1384 send_vblank_event(dev, e, seq, now);
1385 vblwait->reply.sequence = seq;
1386 } else {
1387
1388 list_add_tail(&e->base.link, &dev->vblank_event_list);
1389 vblwait->reply.sequence = req_seq;
1390 }
1391
1392 spin_unlock_irqrestore(&dev->event_lock, flags);
1393
1394 return 0;
1395
1396err_unlock:
1397 spin_unlock_irqrestore(&dev->event_lock, flags);
1398 kfree(e);
1399err_put:
1400 drm_vblank_put(dev, pipe);
1401 return ret;
1402}
1403
1404static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1405{
1406 if (vblwait->request.sequence)
1407 return false;
1408
1409 return _DRM_VBLANK_RELATIVE ==
1410 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1411 _DRM_VBLANK_EVENT |
1412 _DRM_VBLANK_NEXTONMISS));
1413}
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426static u64 widen_32_to_64(u32 narrow, u64 near)
1427{
1428 return near + (s32) (narrow - near);
1429}
1430
1431static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1432 struct drm_wait_vblank_reply *reply)
1433{
1434 ktime_t now;
1435 struct timespec64 ts;
1436
1437
1438
1439
1440
1441
1442 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1443 ts = ktime_to_timespec64(now);
1444 reply->tval_sec = (u32)ts.tv_sec;
1445 reply->tval_usec = ts.tv_nsec / 1000;
1446}
1447
1448int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1449 struct drm_file *file_priv)
1450{
1451 struct drm_crtc *crtc;
1452 struct drm_vblank_crtc *vblank;
1453 union drm_wait_vblank *vblwait = data;
1454 int ret;
1455 u64 req_seq, seq;
1456 unsigned int pipe_index;
1457 unsigned int flags, pipe, high_pipe;
1458
1459 if (!dev->irq_enabled)
1460 return -EINVAL;
1461
1462 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1463 return -EINVAL;
1464
1465 if (vblwait->request.type &
1466 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1467 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1468 DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
1469 vblwait->request.type,
1470 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1471 _DRM_VBLANK_HIGH_CRTC_MASK));
1472 return -EINVAL;
1473 }
1474
1475 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1476 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1477 if (high_pipe)
1478 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1479 else
1480 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1481
1482
1483 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1484 pipe = 0;
1485 drm_for_each_crtc(crtc, dev) {
1486 if (drm_lease_held(file_priv, crtc->base.id)) {
1487 if (pipe_index == 0)
1488 break;
1489 pipe_index--;
1490 }
1491 pipe++;
1492 }
1493 } else {
1494 pipe = pipe_index;
1495 }
1496
1497 if (pipe >= dev->num_crtcs)
1498 return -EINVAL;
1499
1500 vblank = &dev->vblank[pipe];
1501
1502
1503
1504
1505 if (dev->vblank_disable_immediate &&
1506 drm_wait_vblank_is_query(vblwait) &&
1507 READ_ONCE(vblank->enabled)) {
1508 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1509 return 0;
1510 }
1511
1512 ret = drm_vblank_get(dev, pipe);
1513 if (ret) {
1514 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1515 return ret;
1516 }
1517 seq = drm_vblank_count(dev, pipe);
1518
1519 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1520 case _DRM_VBLANK_RELATIVE:
1521 req_seq = seq + vblwait->request.sequence;
1522 vblwait->request.sequence = req_seq;
1523 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1524 break;
1525 case _DRM_VBLANK_ABSOLUTE:
1526 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1527 break;
1528 default:
1529 ret = -EINVAL;
1530 goto done;
1531 }
1532
1533 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1534 vblank_passed(seq, req_seq)) {
1535 req_seq = seq + 1;
1536 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1537 vblwait->request.sequence = req_seq;
1538 }
1539
1540 if (flags & _DRM_VBLANK_EVENT) {
1541
1542
1543
1544 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1545 }
1546
1547 if (req_seq != seq) {
1548 DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1549 req_seq, pipe);
1550 DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
1551 vblank_passed(drm_vblank_count(dev, pipe),
1552 req_seq) ||
1553 !READ_ONCE(vblank->enabled));
1554 }
1555
1556 if (ret != -EINTR) {
1557 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1558
1559 DRM_DEBUG("crtc %d returning %u to client\n",
1560 pipe, vblwait->reply.sequence);
1561 } else {
1562 DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1563 }
1564
1565done:
1566 drm_vblank_put(dev, pipe);
1567 return ret;
1568}
1569
1570static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1571{
1572 struct drm_pending_vblank_event *e, *t;
1573 ktime_t now;
1574 u64 seq;
1575
1576 assert_spin_locked(&dev->event_lock);
1577
1578 seq = drm_vblank_count_and_time(dev, pipe, &now);
1579
1580 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1581 if (e->pipe != pipe)
1582 continue;
1583 if (!vblank_passed(seq, e->sequence))
1584 continue;
1585
1586 DRM_DEBUG("vblank event on %llu, current %llu\n",
1587 e->sequence, seq);
1588
1589 list_del(&e->base.link);
1590 drm_vblank_put(dev, pipe);
1591 send_vblank_event(dev, e, seq, now);
1592 }
1593
1594 trace_drm_vblank_event(pipe, seq);
1595}
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1608{
1609 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1610 unsigned long irqflags;
1611 bool disable_irq;
1612
1613 if (WARN_ON_ONCE(!dev->num_crtcs))
1614 return false;
1615
1616 if (WARN_ON(pipe >= dev->num_crtcs))
1617 return false;
1618
1619 spin_lock_irqsave(&dev->event_lock, irqflags);
1620
1621
1622
1623
1624
1625 spin_lock(&dev->vblank_time_lock);
1626
1627
1628 if (!vblank->enabled) {
1629 spin_unlock(&dev->vblank_time_lock);
1630 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1631 return false;
1632 }
1633
1634 drm_update_vblank_count(dev, pipe, true);
1635
1636 spin_unlock(&dev->vblank_time_lock);
1637
1638 wake_up(&vblank->queue);
1639
1640
1641
1642
1643
1644
1645 disable_irq = (dev->vblank_disable_immediate &&
1646 drm_vblank_offdelay > 0 &&
1647 !atomic_read(&vblank->refcount));
1648
1649 drm_handle_vblank_events(dev, pipe);
1650
1651 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1652
1653 if (disable_irq)
1654 vblank_disable_fn(&vblank->disable_timer);
1655
1656 return true;
1657}
1658EXPORT_SYMBOL(drm_handle_vblank);
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1673{
1674 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1675}
1676EXPORT_SYMBOL(drm_crtc_handle_vblank);
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1687 struct drm_file *file_priv)
1688{
1689 struct drm_crtc *crtc;
1690 struct drm_vblank_crtc *vblank;
1691 int pipe;
1692 struct drm_crtc_get_sequence *get_seq = data;
1693 ktime_t now;
1694 bool vblank_enabled;
1695 int ret;
1696
1697 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1698 return -EINVAL;
1699
1700 if (!dev->irq_enabled)
1701 return -EINVAL;
1702
1703 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1704 if (!crtc)
1705 return -ENOENT;
1706
1707 pipe = drm_crtc_index(crtc);
1708
1709 vblank = &dev->vblank[pipe];
1710 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1711
1712 if (!vblank_enabled) {
1713 ret = drm_crtc_vblank_get(crtc);
1714 if (ret) {
1715 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1716 return ret;
1717 }
1718 }
1719 drm_modeset_lock(&crtc->mutex, NULL);
1720 if (crtc->state)
1721 get_seq->active = crtc->state->enable;
1722 else
1723 get_seq->active = crtc->enabled;
1724 drm_modeset_unlock(&crtc->mutex);
1725 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1726 get_seq->sequence_ns = ktime_to_ns(now);
1727 if (!vblank_enabled)
1728 drm_crtc_vblank_put(crtc);
1729 return 0;
1730}
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1741 struct drm_file *file_priv)
1742{
1743 struct drm_crtc *crtc;
1744 struct drm_vblank_crtc *vblank;
1745 int pipe;
1746 struct drm_crtc_queue_sequence *queue_seq = data;
1747 ktime_t now;
1748 struct drm_pending_vblank_event *e;
1749 u32 flags;
1750 u64 seq;
1751 u64 req_seq;
1752 int ret;
1753 unsigned long spin_flags;
1754
1755 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1756 return -EINVAL;
1757
1758 if (!dev->irq_enabled)
1759 return -EINVAL;
1760
1761 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1762 if (!crtc)
1763 return -ENOENT;
1764
1765 flags = queue_seq->flags;
1766
1767 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1768 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1769 return -EINVAL;
1770
1771 pipe = drm_crtc_index(crtc);
1772
1773 vblank = &dev->vblank[pipe];
1774
1775 e = kzalloc(sizeof(*e), GFP_KERNEL);
1776 if (e == NULL)
1777 return -ENOMEM;
1778
1779 ret = drm_crtc_vblank_get(crtc);
1780 if (ret) {
1781 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1782 goto err_free;
1783 }
1784
1785 seq = drm_vblank_count_and_time(dev, pipe, &now);
1786 req_seq = queue_seq->sequence;
1787
1788 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1789 req_seq += seq;
1790
1791 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1792 req_seq = seq + 1;
1793
1794 e->pipe = pipe;
1795 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1796 e->event.base.length = sizeof(e->event.seq);
1797 e->event.seq.user_data = queue_seq->user_data;
1798
1799 spin_lock_irqsave(&dev->event_lock, spin_flags);
1800
1801
1802
1803
1804
1805
1806
1807 if (!READ_ONCE(vblank->enabled)) {
1808 ret = -EINVAL;
1809 goto err_unlock;
1810 }
1811
1812 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1813 &e->event.base);
1814
1815 if (ret)
1816 goto err_unlock;
1817
1818 e->sequence = req_seq;
1819
1820 if (vblank_passed(seq, req_seq)) {
1821 drm_crtc_vblank_put(crtc);
1822 send_vblank_event(dev, e, seq, now);
1823 queue_seq->sequence = seq;
1824 } else {
1825
1826 list_add_tail(&e->base.link, &dev->vblank_event_list);
1827 queue_seq->sequence = req_seq;
1828 }
1829
1830 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1831 return 0;
1832
1833err_unlock:
1834 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1835 drm_crtc_vblank_put(crtc);
1836err_free:
1837 kfree(e);
1838 return ret;
1839}
1840