1
2
3
4
5
6
7
8
9
10
11#include <linux/timekeeper_internal.h>
12#include <linux/module.h>
13#include <linux/interrupt.h>
14#include <linux/percpu.h>
15#include <linux/init.h>
16#include <linux/mm.h>
17#include <linux/sched.h>
18#include <linux/syscore_ops.h>
19#include <linux/clocksource.h>
20#include <linux/jiffies.h>
21#include <linux/time.h>
22#include <linux/tick.h>
23#include <linux/stop_machine.h>
24#include <linux/pvclock_gtod.h>
25
26#include "tick-internal.h"
27#include "ntp_internal.h"
28
29static struct timekeeper timekeeper;
30static DEFINE_RAW_SPINLOCK(timekeeper_lock);
31static seqcount_t timekeeper_seq;
32static struct timekeeper shadow_timekeeper;
33
34
35int __read_mostly timekeeping_suspended;
36
37
38bool __read_mostly persistent_clock_exist = false;
39
40static inline void tk_normalize_xtime(struct timekeeper *tk)
41{
42 while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) {
43 tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift;
44 tk->xtime_sec++;
45 }
46}
47
48static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts)
49{
50 tk->xtime_sec = ts->tv_sec;
51 tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift;
52}
53
54static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts)
55{
56 tk->xtime_sec += ts->tv_sec;
57 tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift;
58 tk_normalize_xtime(tk);
59}
60
61static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm)
62{
63 struct timespec tmp;
64
65
66
67
68
69 set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec,
70 -tk->wall_to_monotonic.tv_nsec);
71 WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64);
72 tk->wall_to_monotonic = wtm;
73 set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
74 tk->offs_real = timespec_to_ktime(tmp);
75 tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tk->tai_offset, 0));
76}
77
78static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
79{
80
81 WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64);
82
83 tk->total_sleep_time = t;
84 tk->offs_boot = timespec_to_ktime(t);
85}
86
87
88
89
90
91
92
93
94
95
96
97static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
98{
99 cycle_t interval;
100 u64 tmp, ntpinterval;
101 struct clocksource *old_clock;
102
103 old_clock = tk->clock;
104 tk->clock = clock;
105 tk->cycle_last = clock->cycle_last = clock->read(clock);
106
107
108 tmp = NTP_INTERVAL_LENGTH;
109 tmp <<= clock->shift;
110 ntpinterval = tmp;
111 tmp += clock->mult/2;
112 do_div(tmp, clock->mult);
113 if (tmp == 0)
114 tmp = 1;
115
116 interval = (cycle_t) tmp;
117 tk->cycle_interval = interval;
118
119
120 tk->xtime_interval = (u64) interval * clock->mult;
121 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
122 tk->raw_interval =
123 ((u64) interval * clock->mult) >> clock->shift;
124
125
126 if (old_clock) {
127 int shift_change = clock->shift - old_clock->shift;
128 if (shift_change < 0)
129 tk->xtime_nsec >>= -shift_change;
130 else
131 tk->xtime_nsec <<= shift_change;
132 }
133 tk->shift = clock->shift;
134
135 tk->ntp_error = 0;
136 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
137
138
139
140
141
142
143 tk->mult = clock->mult;
144}
145
146
147
148#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
149u32 (*arch_gettimeoffset)(void);
150
151u32 get_arch_timeoffset(void)
152{
153 if (likely(arch_gettimeoffset))
154 return arch_gettimeoffset();
155 return 0;
156}
157#else
158static inline u32 get_arch_timeoffset(void) { return 0; }
159#endif
160
161static inline s64 timekeeping_get_ns(struct timekeeper *tk)
162{
163 cycle_t cycle_now, cycle_delta;
164 struct clocksource *clock;
165 s64 nsec;
166
167
168 clock = tk->clock;
169 cycle_now = clock->read(clock);
170
171
172 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
173
174 nsec = cycle_delta * tk->mult + tk->xtime_nsec;
175 nsec >>= tk->shift;
176
177
178 return nsec + get_arch_timeoffset();
179}
180
181static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
182{
183 cycle_t cycle_now, cycle_delta;
184 struct clocksource *clock;
185 s64 nsec;
186
187
188 clock = tk->clock;
189 cycle_now = clock->read(clock);
190
191
192 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
193
194
195 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
196
197
198 return nsec + get_arch_timeoffset();
199}
200
201static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
202
203static void update_pvclock_gtod(struct timekeeper *tk)
204{
205 raw_notifier_call_chain(&pvclock_gtod_chain, 0, tk);
206}
207
208
209
210
211int pvclock_gtod_register_notifier(struct notifier_block *nb)
212{
213 struct timekeeper *tk = &timekeeper;
214 unsigned long flags;
215 int ret;
216
217 raw_spin_lock_irqsave(&timekeeper_lock, flags);
218 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
219 update_pvclock_gtod(tk);
220 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
221
222 return ret;
223}
224EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
225
226
227
228
229
230int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
231{
232 unsigned long flags;
233 int ret;
234
235 raw_spin_lock_irqsave(&timekeeper_lock, flags);
236 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
237 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
238
239 return ret;
240}
241EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
242
243
244static void timekeeping_update(struct timekeeper *tk, bool clearntp, bool mirror)
245{
246 if (clearntp) {
247 tk->ntp_error = 0;
248 ntp_clear();
249 }
250 update_vsyscall(tk);
251 update_pvclock_gtod(tk);
252
253 if (mirror)
254 memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));
255}
256
257
258
259
260
261
262
263
264static void timekeeping_forward_now(struct timekeeper *tk)
265{
266 cycle_t cycle_now, cycle_delta;
267 struct clocksource *clock;
268 s64 nsec;
269
270 clock = tk->clock;
271 cycle_now = clock->read(clock);
272 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
273 tk->cycle_last = clock->cycle_last = cycle_now;
274
275 tk->xtime_nsec += cycle_delta * tk->mult;
276
277
278 tk->xtime_nsec += (u64)get_arch_timeoffset() << tk->shift;
279
280 tk_normalize_xtime(tk);
281
282 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
283 timespec_add_ns(&tk->raw_time, nsec);
284}
285
286
287
288
289
290
291
292
293int __getnstimeofday(struct timespec *ts)
294{
295 struct timekeeper *tk = &timekeeper;
296 unsigned long seq;
297 s64 nsecs = 0;
298
299 do {
300 seq = read_seqcount_begin(&timekeeper_seq);
301
302 ts->tv_sec = tk->xtime_sec;
303 nsecs = timekeeping_get_ns(tk);
304
305 } while (read_seqcount_retry(&timekeeper_seq, seq));
306
307 ts->tv_nsec = 0;
308 timespec_add_ns(ts, nsecs);
309
310
311
312
313
314 if (unlikely(timekeeping_suspended))
315 return -EAGAIN;
316 return 0;
317}
318EXPORT_SYMBOL(__getnstimeofday);
319
320
321
322
323
324
325
326void getnstimeofday(struct timespec *ts)
327{
328 WARN_ON(__getnstimeofday(ts));
329}
330EXPORT_SYMBOL(getnstimeofday);
331
332ktime_t ktime_get(void)
333{
334 struct timekeeper *tk = &timekeeper;
335 unsigned int seq;
336 s64 secs, nsecs;
337
338 WARN_ON(timekeeping_suspended);
339
340 do {
341 seq = read_seqcount_begin(&timekeeper_seq);
342 secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
343 nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;
344
345 } while (read_seqcount_retry(&timekeeper_seq, seq));
346
347
348
349
350 return ktime_add_ns(ktime_set(secs, 0), nsecs);
351}
352EXPORT_SYMBOL_GPL(ktime_get);
353
354
355
356
357
358
359
360
361
362void ktime_get_ts(struct timespec *ts)
363{
364 struct timekeeper *tk = &timekeeper;
365 struct timespec tomono;
366 s64 nsec;
367 unsigned int seq;
368
369 WARN_ON(timekeeping_suspended);
370
371 do {
372 seq = read_seqcount_begin(&timekeeper_seq);
373 ts->tv_sec = tk->xtime_sec;
374 nsec = timekeeping_get_ns(tk);
375 tomono = tk->wall_to_monotonic;
376
377 } while (read_seqcount_retry(&timekeeper_seq, seq));
378
379 ts->tv_sec += tomono.tv_sec;
380 ts->tv_nsec = 0;
381 timespec_add_ns(ts, nsec + tomono.tv_nsec);
382}
383EXPORT_SYMBOL_GPL(ktime_get_ts);
384
385
386
387
388
389
390
391
392void timekeeping_clocktai(struct timespec *ts)
393{
394 struct timekeeper *tk = &timekeeper;
395 unsigned long seq;
396 u64 nsecs;
397
398 WARN_ON(timekeeping_suspended);
399
400 do {
401 seq = read_seqcount_begin(&timekeeper_seq);
402
403 ts->tv_sec = tk->xtime_sec + tk->tai_offset;
404 nsecs = timekeeping_get_ns(tk);
405
406 } while (read_seqcount_retry(&timekeeper_seq, seq));
407
408 ts->tv_nsec = 0;
409 timespec_add_ns(ts, nsecs);
410
411}
412EXPORT_SYMBOL(timekeeping_clocktai);
413
414
415
416
417
418
419
420ktime_t ktime_get_clocktai(void)
421{
422 struct timespec ts;
423
424 timekeeping_clocktai(&ts);
425 return timespec_to_ktime(ts);
426}
427EXPORT_SYMBOL(ktime_get_clocktai);
428
429#ifdef CONFIG_NTP_PPS
430
431
432
433
434
435
436
437
438
439
440void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
441{
442 struct timekeeper *tk = &timekeeper;
443 unsigned long seq;
444 s64 nsecs_raw, nsecs_real;
445
446 WARN_ON_ONCE(timekeeping_suspended);
447
448 do {
449 seq = read_seqcount_begin(&timekeeper_seq);
450
451 *ts_raw = tk->raw_time;
452 ts_real->tv_sec = tk->xtime_sec;
453 ts_real->tv_nsec = 0;
454
455 nsecs_raw = timekeeping_get_ns_raw(tk);
456 nsecs_real = timekeeping_get_ns(tk);
457
458 } while (read_seqcount_retry(&timekeeper_seq, seq));
459
460 timespec_add_ns(ts_raw, nsecs_raw);
461 timespec_add_ns(ts_real, nsecs_real);
462}
463EXPORT_SYMBOL(getnstime_raw_and_real);
464
465#endif
466
467
468
469
470
471
472
473void do_gettimeofday(struct timeval *tv)
474{
475 struct timespec now;
476
477 getnstimeofday(&now);
478 tv->tv_sec = now.tv_sec;
479 tv->tv_usec = now.tv_nsec/1000;
480}
481EXPORT_SYMBOL(do_gettimeofday);
482
483
484
485
486
487
488
489int do_settimeofday(const struct timespec *tv)
490{
491 struct timekeeper *tk = &timekeeper;
492 struct timespec ts_delta, xt;
493 unsigned long flags;
494
495 if (!timespec_valid_strict(tv))
496 return -EINVAL;
497
498 raw_spin_lock_irqsave(&timekeeper_lock, flags);
499 write_seqcount_begin(&timekeeper_seq);
500
501 timekeeping_forward_now(tk);
502
503 xt = tk_xtime(tk);
504 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
505 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;
506
507 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta));
508
509 tk_set_xtime(tk, tv);
510
511 timekeeping_update(tk, true, true);
512
513 write_seqcount_end(&timekeeper_seq);
514 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
515
516
517 clock_was_set();
518
519 return 0;
520}
521EXPORT_SYMBOL(do_settimeofday);
522
523
524
525
526
527
528
529int timekeeping_inject_offset(struct timespec *ts)
530{
531 struct timekeeper *tk = &timekeeper;
532 unsigned long flags;
533 struct timespec tmp;
534 int ret = 0;
535
536 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
537 return -EINVAL;
538
539 raw_spin_lock_irqsave(&timekeeper_lock, flags);
540 write_seqcount_begin(&timekeeper_seq);
541
542 timekeeping_forward_now(tk);
543
544
545 tmp = timespec_add(tk_xtime(tk), *ts);
546 if (!timespec_valid_strict(&tmp)) {
547 ret = -EINVAL;
548 goto error;
549 }
550
551 tk_xtime_add(tk, ts);
552 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts));
553
554error:
555 timekeeping_update(tk, true, true);
556
557 write_seqcount_end(&timekeeper_seq);
558 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
559
560
561 clock_was_set();
562
563 return ret;
564}
565EXPORT_SYMBOL(timekeeping_inject_offset);
566
567
568
569
570
571
572s32 timekeeping_get_tai_offset(void)
573{
574 struct timekeeper *tk = &timekeeper;
575 unsigned int seq;
576 s32 ret;
577
578 do {
579 seq = read_seqcount_begin(&timekeeper_seq);
580 ret = tk->tai_offset;
581 } while (read_seqcount_retry(&timekeeper_seq, seq));
582
583 return ret;
584}
585
586
587
588
589
590static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
591{
592 tk->tai_offset = tai_offset;
593 tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tai_offset, 0));
594}
595
596
597
598
599
600void timekeeping_set_tai_offset(s32 tai_offset)
601{
602 struct timekeeper *tk = &timekeeper;
603 unsigned long flags;
604
605 raw_spin_lock_irqsave(&timekeeper_lock, flags);
606 write_seqcount_begin(&timekeeper_seq);
607 __timekeeping_set_tai_offset(tk, tai_offset);
608 write_seqcount_end(&timekeeper_seq);
609 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
610 clock_was_set();
611}
612
613
614
615
616
617
618static int change_clocksource(void *data)
619{
620 struct timekeeper *tk = &timekeeper;
621 struct clocksource *new, *old;
622 unsigned long flags;
623
624 new = (struct clocksource *) data;
625
626 raw_spin_lock_irqsave(&timekeeper_lock, flags);
627 write_seqcount_begin(&timekeeper_seq);
628
629 timekeeping_forward_now(tk);
630 if (!new->enable || new->enable(new) == 0) {
631 old = tk->clock;
632 tk_setup_internals(tk, new);
633 if (old->disable)
634 old->disable(old);
635 }
636 timekeeping_update(tk, true, true);
637
638 write_seqcount_end(&timekeeper_seq);
639 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
640
641 return 0;
642}
643
644
645
646
647
648
649
650
651void timekeeping_notify(struct clocksource *clock)
652{
653 struct timekeeper *tk = &timekeeper;
654
655 if (tk->clock == clock)
656 return;
657 stop_machine(change_clocksource, clock, NULL);
658 tick_clock_notify();
659}
660
661
662
663
664
665
666ktime_t ktime_get_real(void)
667{
668 struct timespec now;
669
670 getnstimeofday(&now);
671
672 return timespec_to_ktime(now);
673}
674EXPORT_SYMBOL_GPL(ktime_get_real);
675
676
677
678
679
680
681
682void getrawmonotonic(struct timespec *ts)
683{
684 struct timekeeper *tk = &timekeeper;
685 unsigned long seq;
686 s64 nsecs;
687
688 do {
689 seq = read_seqcount_begin(&timekeeper_seq);
690 nsecs = timekeeping_get_ns_raw(tk);
691 *ts = tk->raw_time;
692
693 } while (read_seqcount_retry(&timekeeper_seq, seq));
694
695 timespec_add_ns(ts, nsecs);
696}
697EXPORT_SYMBOL(getrawmonotonic);
698
699
700
701
702int timekeeping_valid_for_hres(void)
703{
704 struct timekeeper *tk = &timekeeper;
705 unsigned long seq;
706 int ret;
707
708 do {
709 seq = read_seqcount_begin(&timekeeper_seq);
710
711 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
712
713 } while (read_seqcount_retry(&timekeeper_seq, seq));
714
715 return ret;
716}
717
718
719
720
721u64 timekeeping_max_deferment(void)
722{
723 struct timekeeper *tk = &timekeeper;
724 unsigned long seq;
725 u64 ret;
726
727 do {
728 seq = read_seqcount_begin(&timekeeper_seq);
729
730 ret = tk->clock->max_idle_ns;
731
732 } while (read_seqcount_retry(&timekeeper_seq, seq));
733
734 return ret;
735}
736
737
738
739
740
741
742
743
744
745
746void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
747{
748 ts->tv_sec = 0;
749 ts->tv_nsec = 0;
750}
751
752
753
754
755
756
757
758
759
760
761void __attribute__((weak)) read_boot_clock(struct timespec *ts)
762{
763 ts->tv_sec = 0;
764 ts->tv_nsec = 0;
765}
766
767
768
769
770void __init timekeeping_init(void)
771{
772 struct timekeeper *tk = &timekeeper;
773 struct clocksource *clock;
774 unsigned long flags;
775 struct timespec now, boot, tmp;
776
777 read_persistent_clock(&now);
778
779 if (!timespec_valid_strict(&now)) {
780 pr_warn("WARNING: Persistent clock returned invalid value!\n"
781 " Check your CMOS/BIOS settings.\n");
782 now.tv_sec = 0;
783 now.tv_nsec = 0;
784 } else if (now.tv_sec || now.tv_nsec)
785 persistent_clock_exist = true;
786
787 read_boot_clock(&boot);
788 if (!timespec_valid_strict(&boot)) {
789 pr_warn("WARNING: Boot clock returned invalid value!\n"
790 " Check your CMOS/BIOS settings.\n");
791 boot.tv_sec = 0;
792 boot.tv_nsec = 0;
793 }
794
795 raw_spin_lock_irqsave(&timekeeper_lock, flags);
796 write_seqcount_begin(&timekeeper_seq);
797 ntp_init();
798
799 clock = clocksource_default_clock();
800 if (clock->enable)
801 clock->enable(clock);
802 tk_setup_internals(tk, clock);
803
804 tk_set_xtime(tk, &now);
805 tk->raw_time.tv_sec = 0;
806 tk->raw_time.tv_nsec = 0;
807 if (boot.tv_sec == 0 && boot.tv_nsec == 0)
808 boot = tk_xtime(tk);
809
810 set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec);
811 tk_set_wall_to_mono(tk, tmp);
812
813 tmp.tv_sec = 0;
814 tmp.tv_nsec = 0;
815 tk_set_sleep_time(tk, tmp);
816
817 memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));
818
819 write_seqcount_end(&timekeeper_seq);
820 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
821}
822
823
824static struct timespec timekeeping_suspend_time;
825
826
827
828
829
830
831
832
833static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
834 struct timespec *delta)
835{
836 if (!timespec_valid_strict(delta)) {
837 printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
838 "sleep delta value!\n");
839 return;
840 }
841 tk_xtime_add(tk, delta);
842 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta));
843 tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta));
844}
845
846
847
848
849
850
851
852
853
854
855
856void timekeeping_inject_sleeptime(struct timespec *delta)
857{
858 struct timekeeper *tk = &timekeeper;
859 unsigned long flags;
860
861
862
863
864
865 if (has_persistent_clock())
866 return;
867
868 raw_spin_lock_irqsave(&timekeeper_lock, flags);
869 write_seqcount_begin(&timekeeper_seq);
870
871 timekeeping_forward_now(tk);
872
873 __timekeeping_inject_sleeptime(tk, delta);
874
875 timekeeping_update(tk, true, true);
876
877 write_seqcount_end(&timekeeper_seq);
878 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
879
880
881 clock_was_set();
882}
883
884
885
886
887
888
889
890
891static void timekeeping_resume(void)
892{
893 struct timekeeper *tk = &timekeeper;
894 struct clocksource *clock = tk->clock;
895 unsigned long flags;
896 struct timespec ts_new, ts_delta;
897 cycle_t cycle_now, cycle_delta;
898 bool suspendtime_found = false;
899
900 read_persistent_clock(&ts_new);
901
902 clockevents_resume();
903 clocksource_resume();
904
905 raw_spin_lock_irqsave(&timekeeper_lock, flags);
906 write_seqcount_begin(&timekeeper_seq);
907
908
909
910
911
912
913
914
915
916
917
918
919
920 cycle_now = clock->read(clock);
921 if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
922 cycle_now > clock->cycle_last) {
923 u64 num, max = ULLONG_MAX;
924 u32 mult = clock->mult;
925 u32 shift = clock->shift;
926 s64 nsec = 0;
927
928 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
929
930
931
932
933
934
935 do_div(max, mult);
936 if (cycle_delta > max) {
937 num = div64_u64(cycle_delta, max);
938 nsec = (((u64) max * mult) >> shift) * num;
939 cycle_delta -= num * max;
940 }
941 nsec += ((u64) cycle_delta * mult) >> shift;
942
943 ts_delta = ns_to_timespec(nsec);
944 suspendtime_found = true;
945 } else if (timespec_compare(&ts_new, &timekeeping_suspend_time) > 0) {
946 ts_delta = timespec_sub(ts_new, timekeeping_suspend_time);
947 suspendtime_found = true;
948 }
949
950 if (suspendtime_found)
951 __timekeeping_inject_sleeptime(tk, &ts_delta);
952
953
954 tk->cycle_last = clock->cycle_last = cycle_now;
955 tk->ntp_error = 0;
956 timekeeping_suspended = 0;
957 timekeeping_update(tk, false, true);
958 write_seqcount_end(&timekeeper_seq);
959 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
960
961 touch_softlockup_watchdog();
962
963 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
964
965
966 hrtimers_resume();
967}
968
969static int timekeeping_suspend(void)
970{
971 struct timekeeper *tk = &timekeeper;
972 unsigned long flags;
973 struct timespec delta, delta_delta;
974 static struct timespec old_delta;
975
976 read_persistent_clock(&timekeeping_suspend_time);
977
978
979
980
981
982
983 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
984 persistent_clock_exist = true;
985
986 raw_spin_lock_irqsave(&timekeeper_lock, flags);
987 write_seqcount_begin(&timekeeper_seq);
988 timekeeping_forward_now(tk);
989 timekeeping_suspended = 1;
990
991
992
993
994
995
996
997 delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time);
998 delta_delta = timespec_sub(delta, old_delta);
999 if (abs(delta_delta.tv_sec) >= 2) {
1000
1001
1002
1003
1004 old_delta = delta;
1005 } else {
1006
1007 timekeeping_suspend_time =
1008 timespec_add(timekeeping_suspend_time, delta_delta);
1009 }
1010 write_seqcount_end(&timekeeper_seq);
1011 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1012
1013 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
1014 clocksource_suspend();
1015 clockevents_suspend();
1016
1017 return 0;
1018}
1019
1020
1021static struct syscore_ops timekeeping_syscore_ops = {
1022 .resume = timekeeping_resume,
1023 .suspend = timekeeping_suspend,
1024};
1025
1026static int __init timekeeping_init_ops(void)
1027{
1028 register_syscore_ops(&timekeeping_syscore_ops);
1029 return 0;
1030}
1031
1032device_initcall(timekeeping_init_ops);
1033
1034
1035
1036
1037
1038static __always_inline int timekeeping_bigadjust(struct timekeeper *tk,
1039 s64 error, s64 *interval,
1040 s64 *offset)
1041{
1042 s64 tick_error, i;
1043 u32 look_ahead, adj;
1044 s32 error2, mult;
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055 error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
1056 error2 = abs(error2);
1057 for (look_ahead = 0; error2 > 0; look_ahead++)
1058 error2 >>= 2;
1059
1060
1061
1062
1063
1064 tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1);
1065 tick_error -= tk->xtime_interval >> 1;
1066 error = ((error - tick_error) >> look_ahead) + tick_error;
1067
1068
1069 i = *interval;
1070 mult = 1;
1071 if (error < 0) {
1072 error = -error;
1073 *interval = -*interval;
1074 *offset = -*offset;
1075 mult = -1;
1076 }
1077 for (adj = 0; error > i; adj++)
1078 error >>= 1;
1079
1080 *interval <<= adj;
1081 *offset <<= adj;
1082 return mult << adj;
1083}
1084
1085
1086
1087
1088
1089
1090static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1091{
1092 s64 error, interval = tk->cycle_interval;
1093 int adj;
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108 error = tk->ntp_error >> (tk->ntp_error_shift - 1);
1109 if (error > interval) {
1110
1111
1112
1113
1114
1115
1116 error >>= 2;
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127 if (likely(error <= interval))
1128 adj = 1;
1129 else
1130 adj = timekeeping_bigadjust(tk, error, &interval, &offset);
1131 } else {
1132 if (error < -interval) {
1133
1134 error >>= 2;
1135 if (likely(error >= -interval)) {
1136 adj = -1;
1137 interval = -interval;
1138 offset = -offset;
1139 } else {
1140 adj = timekeeping_bigadjust(tk, error, &interval, &offset);
1141 }
1142 } else {
1143 goto out_adjust;
1144 }
1145 }
1146
1147 if (unlikely(tk->clock->maxadj &&
1148 (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) {
1149 printk_once(KERN_WARNING
1150 "Adjusting %s more than 11%% (%ld vs %ld)\n",
1151 tk->clock->name, (long)tk->mult + adj,
1152 (long)tk->clock->mult + tk->clock->maxadj);
1153 }
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203 tk->mult += adj;
1204 tk->xtime_interval += interval;
1205 tk->xtime_nsec -= offset;
1206 tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
1207
1208out_adjust:
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223 if (unlikely((s64)tk->xtime_nsec < 0)) {
1224 s64 neg = -(s64)tk->xtime_nsec;
1225 tk->xtime_nsec = 0;
1226 tk->ntp_error += neg << tk->ntp_error_shift;
1227 }
1228
1229}
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239static inline void accumulate_nsecs_to_secs(struct timekeeper *tk)
1240{
1241 u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
1242
1243 while (tk->xtime_nsec >= nsecps) {
1244 int leap;
1245
1246 tk->xtime_nsec -= nsecps;
1247 tk->xtime_sec++;
1248
1249
1250 leap = second_overflow(tk->xtime_sec);
1251 if (unlikely(leap)) {
1252 struct timespec ts;
1253
1254 tk->xtime_sec += leap;
1255
1256 ts.tv_sec = leap;
1257 ts.tv_nsec = 0;
1258 tk_set_wall_to_mono(tk,
1259 timespec_sub(tk->wall_to_monotonic, ts));
1260
1261 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1262
1263 clock_was_set_delayed();
1264 }
1265 }
1266}
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1278 u32 shift)
1279{
1280 cycle_t interval = tk->cycle_interval << shift;
1281 u64 raw_nsecs;
1282
1283
1284 if (offset < interval)
1285 return offset;
1286
1287
1288 offset -= interval;
1289 tk->cycle_last += interval;
1290
1291 tk->xtime_nsec += tk->xtime_interval << shift;
1292 accumulate_nsecs_to_secs(tk);
1293
1294
1295 raw_nsecs = (u64)tk->raw_interval << shift;
1296 raw_nsecs += tk->raw_time.tv_nsec;
1297 if (raw_nsecs >= NSEC_PER_SEC) {
1298 u64 raw_secs = raw_nsecs;
1299 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
1300 tk->raw_time.tv_sec += raw_secs;
1301 }
1302 tk->raw_time.tv_nsec = raw_nsecs;
1303
1304
1305 tk->ntp_error += ntp_tick_length() << shift;
1306 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
1307 (tk->ntp_error_shift + shift);
1308
1309 return offset;
1310}
1311
1312#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
1313static inline void old_vsyscall_fixup(struct timekeeper *tk)
1314{
1315 s64 remainder;
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327 remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
1328 tk->xtime_nsec -= remainder;
1329 tk->xtime_nsec += 1ULL << tk->shift;
1330 tk->ntp_error += remainder << tk->ntp_error_shift;
1331
1332}
1333#else
1334#define old_vsyscall_fixup(tk)
1335#endif
1336
1337
1338
1339
1340
1341
1342
1343static void update_wall_time(void)
1344{
1345 struct clocksource *clock;
1346 struct timekeeper *real_tk = &timekeeper;
1347 struct timekeeper *tk = &shadow_timekeeper;
1348 cycle_t offset;
1349 int shift = 0, maxshift;
1350 unsigned long flags;
1351
1352 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1353
1354
1355 if (unlikely(timekeeping_suspended))
1356 goto out;
1357
1358 clock = real_tk->clock;
1359
1360#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
1361 offset = real_tk->cycle_interval;
1362#else
1363 offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
1364#endif
1365
1366
1367 if (offset < real_tk->cycle_interval)
1368 goto out;
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378 shift = ilog2(offset) - ilog2(tk->cycle_interval);
1379 shift = max(0, shift);
1380
1381 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
1382 shift = min(shift, maxshift);
1383 while (offset >= tk->cycle_interval) {
1384 offset = logarithmic_accumulation(tk, offset, shift);
1385 if (offset < tk->cycle_interval<<shift)
1386 shift--;
1387 }
1388
1389
1390 timekeeping_adjust(tk, offset);
1391
1392
1393
1394
1395
1396 old_vsyscall_fixup(tk);
1397
1398
1399
1400
1401
1402 accumulate_nsecs_to_secs(tk);
1403
1404 write_seqcount_begin(&timekeeper_seq);
1405
1406 clock->cycle_last = tk->cycle_last;
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417 memcpy(real_tk, tk, sizeof(*tk));
1418 timekeeping_update(real_tk, false, false);
1419 write_seqcount_end(&timekeeper_seq);
1420out:
1421 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1422}
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435void getboottime(struct timespec *ts)
1436{
1437 struct timekeeper *tk = &timekeeper;
1438 struct timespec boottime = {
1439 .tv_sec = tk->wall_to_monotonic.tv_sec +
1440 tk->total_sleep_time.tv_sec,
1441 .tv_nsec = tk->wall_to_monotonic.tv_nsec +
1442 tk->total_sleep_time.tv_nsec
1443 };
1444
1445 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
1446}
1447EXPORT_SYMBOL_GPL(getboottime);
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458void get_monotonic_boottime(struct timespec *ts)
1459{
1460 struct timekeeper *tk = &timekeeper;
1461 struct timespec tomono, sleep;
1462 s64 nsec;
1463 unsigned int seq;
1464
1465 WARN_ON(timekeeping_suspended);
1466
1467 do {
1468 seq = read_seqcount_begin(&timekeeper_seq);
1469 ts->tv_sec = tk->xtime_sec;
1470 nsec = timekeeping_get_ns(tk);
1471 tomono = tk->wall_to_monotonic;
1472 sleep = tk->total_sleep_time;
1473
1474 } while (read_seqcount_retry(&timekeeper_seq, seq));
1475
1476 ts->tv_sec += tomono.tv_sec + sleep.tv_sec;
1477 ts->tv_nsec = 0;
1478 timespec_add_ns(ts, nsec + tomono.tv_nsec + sleep.tv_nsec);
1479}
1480EXPORT_SYMBOL_GPL(get_monotonic_boottime);
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490ktime_t ktime_get_boottime(void)
1491{
1492 struct timespec ts;
1493
1494 get_monotonic_boottime(&ts);
1495 return timespec_to_ktime(ts);
1496}
1497EXPORT_SYMBOL_GPL(ktime_get_boottime);
1498
1499
1500
1501
1502
1503void monotonic_to_bootbased(struct timespec *ts)
1504{
1505 struct timekeeper *tk = &timekeeper;
1506
1507 *ts = timespec_add(*ts, tk->total_sleep_time);
1508}
1509EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
1510
1511unsigned long get_seconds(void)
1512{
1513 struct timekeeper *tk = &timekeeper;
1514
1515 return tk->xtime_sec;
1516}
1517EXPORT_SYMBOL(get_seconds);
1518
1519struct timespec __current_kernel_time(void)
1520{
1521 struct timekeeper *tk = &timekeeper;
1522
1523 return tk_xtime(tk);
1524}
1525
1526struct timespec current_kernel_time(void)
1527{
1528 struct timekeeper *tk = &timekeeper;
1529 struct timespec now;
1530 unsigned long seq;
1531
1532 do {
1533 seq = read_seqcount_begin(&timekeeper_seq);
1534
1535 now = tk_xtime(tk);
1536 } while (read_seqcount_retry(&timekeeper_seq, seq));
1537
1538 return now;
1539}
1540EXPORT_SYMBOL(current_kernel_time);
1541
1542struct timespec get_monotonic_coarse(void)
1543{
1544 struct timekeeper *tk = &timekeeper;
1545 struct timespec now, mono;
1546 unsigned long seq;
1547
1548 do {
1549 seq = read_seqcount_begin(&timekeeper_seq);
1550
1551 now = tk_xtime(tk);
1552 mono = tk->wall_to_monotonic;
1553 } while (read_seqcount_retry(&timekeeper_seq, seq));
1554
1555 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
1556 now.tv_nsec + mono.tv_nsec);
1557 return now;
1558}
1559
1560
1561
1562
1563void do_timer(unsigned long ticks)
1564{
1565 jiffies_64 += ticks;
1566 update_wall_time();
1567 calc_global_load(ticks);
1568}
1569
1570
1571
1572
1573
1574
1575
1576
1577void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1578 struct timespec *wtom, struct timespec *sleep)
1579{
1580 struct timekeeper *tk = &timekeeper;
1581 unsigned long seq;
1582
1583 do {
1584 seq = read_seqcount_begin(&timekeeper_seq);
1585 *xtim = tk_xtime(tk);
1586 *wtom = tk->wall_to_monotonic;
1587 *sleep = tk->total_sleep_time;
1588 } while (read_seqcount_retry(&timekeeper_seq, seq));
1589}
1590
1591#ifdef CONFIG_HIGH_RES_TIMERS
1592
1593
1594
1595
1596
1597
1598
1599
1600ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot,
1601 ktime_t *offs_tai)
1602{
1603 struct timekeeper *tk = &timekeeper;
1604 ktime_t now;
1605 unsigned int seq;
1606 u64 secs, nsecs;
1607
1608 do {
1609 seq = read_seqcount_begin(&timekeeper_seq);
1610
1611 secs = tk->xtime_sec;
1612 nsecs = timekeeping_get_ns(tk);
1613
1614 *offs_real = tk->offs_real;
1615 *offs_boot = tk->offs_boot;
1616 *offs_tai = tk->offs_tai;
1617 } while (read_seqcount_retry(&timekeeper_seq, seq));
1618
1619 now = ktime_add_ns(ktime_set(secs, 0), nsecs);
1620 now = ktime_sub(now, *offs_real);
1621 return now;
1622}
1623#endif
1624
1625
1626
1627
1628ktime_t ktime_get_monotonic_offset(void)
1629{
1630 struct timekeeper *tk = &timekeeper;
1631 unsigned long seq;
1632 struct timespec wtom;
1633
1634 do {
1635 seq = read_seqcount_begin(&timekeeper_seq);
1636 wtom = tk->wall_to_monotonic;
1637 } while (read_seqcount_retry(&timekeeper_seq, seq));
1638
1639 return timespec_to_ktime(wtom);
1640}
1641EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
1642
1643
1644
1645
1646int do_adjtimex(struct timex *txc)
1647{
1648 struct timekeeper *tk = &timekeeper;
1649 unsigned long flags;
1650 struct timespec ts;
1651 s32 orig_tai, tai;
1652 int ret;
1653
1654
1655 ret = ntp_validate_timex(txc);
1656 if (ret)
1657 return ret;
1658
1659 if (txc->modes & ADJ_SETOFFSET) {
1660 struct timespec delta;
1661 delta.tv_sec = txc->time.tv_sec;
1662 delta.tv_nsec = txc->time.tv_usec;
1663 if (!(txc->modes & ADJ_NANO))
1664 delta.tv_nsec *= 1000;
1665 ret = timekeeping_inject_offset(&delta);
1666 if (ret)
1667 return ret;
1668 }
1669
1670 getnstimeofday(&ts);
1671
1672 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1673 write_seqcount_begin(&timekeeper_seq);
1674
1675 orig_tai = tai = tk->tai_offset;
1676 ret = __do_adjtimex(txc, &ts, &tai);
1677
1678 if (tai != orig_tai) {
1679 __timekeeping_set_tai_offset(tk, tai);
1680 clock_was_set_delayed();
1681 }
1682 write_seqcount_end(&timekeeper_seq);
1683 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1684
1685 return ret;
1686}
1687
1688#ifdef CONFIG_NTP_PPS
1689
1690
1691
1692void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts)
1693{
1694 unsigned long flags;
1695
1696 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1697 write_seqcount_begin(&timekeeper_seq);
1698
1699 __hardpps(phase_ts, raw_ts);
1700
1701 write_seqcount_end(&timekeeper_seq);
1702 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1703}
1704EXPORT_SYMBOL(hardpps);
1705#endif
1706
1707
1708
1709
1710
1711
1712
1713void xtime_update(unsigned long ticks)
1714{
1715 write_seqlock(&jiffies_lock);
1716 do_timer(ticks);
1717 write_sequnlock(&jiffies_lock);
1718}
1719