1
2
3
4
5
6
7
8
9
10
11#include <linux/timekeeper_internal.h>
12#include <linux/module.h>
13#include <linux/interrupt.h>
14#include <linux/percpu.h>
15#include <linux/init.h>
16#include <linux/mm.h>
17#include <linux/sched.h>
18#include <linux/syscore_ops.h>
19#include <linux/clocksource.h>
20#include <linux/jiffies.h>
21#include <linux/time.h>
22#include <linux/tick.h>
23#include <linux/stop_machine.h>
24#include <linux/pvclock_gtod.h>
25
26#include "tick-internal.h"
27#include "ntp_internal.h"
28#include "timekeeping_internal.h"
29
30#define TK_CLEAR_NTP (1 << 0)
31#define TK_MIRROR (1 << 1)
32#define TK_CLOCK_WAS_SET (1 << 2)
33
34static struct timekeeper timekeeper;
35static DEFINE_RAW_SPINLOCK(timekeeper_lock);
36static seqcount_t timekeeper_seq;
37static struct timekeeper shadow_timekeeper;
38
39
40int __read_mostly timekeeping_suspended;
41
42
43bool __read_mostly persistent_clock_exist = false;
44
45static inline void tk_normalize_xtime(struct timekeeper *tk)
46{
47 while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) {
48 tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift;
49 tk->xtime_sec++;
50 }
51}
52
53static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts)
54{
55 tk->xtime_sec = ts->tv_sec;
56 tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift;
57}
58
59static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts)
60{
61 tk->xtime_sec += ts->tv_sec;
62 tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift;
63 tk_normalize_xtime(tk);
64}
65
66static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm)
67{
68 struct timespec tmp;
69
70
71
72
73
74 set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec,
75 -tk->wall_to_monotonic.tv_nsec);
76 WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64);
77 tk->wall_to_monotonic = wtm;
78 set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
79 tk->offs_real = timespec_to_ktime(tmp);
80 tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tk->tai_offset, 0));
81}
82
83static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
84{
85
86 WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64);
87
88 tk->total_sleep_time = t;
89 tk->offs_boot = timespec_to_ktime(t);
90}
91
92
93
94
95
96
97
98
99
100
101
102static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
103{
104 cycle_t interval;
105 u64 tmp, ntpinterval;
106 struct clocksource *old_clock;
107
108 old_clock = tk->clock;
109 tk->clock = clock;
110 tk->cycle_last = clock->cycle_last = clock->read(clock);
111
112
113 tmp = NTP_INTERVAL_LENGTH;
114 tmp <<= clock->shift;
115 ntpinterval = tmp;
116 tmp += clock->mult/2;
117 do_div(tmp, clock->mult);
118 if (tmp == 0)
119 tmp = 1;
120
121 interval = (cycle_t) tmp;
122 tk->cycle_interval = interval;
123
124
125 tk->xtime_interval = (u64) interval * clock->mult;
126 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
127 tk->raw_interval =
128 ((u64) interval * clock->mult) >> clock->shift;
129
130
131 if (old_clock) {
132 int shift_change = clock->shift - old_clock->shift;
133 if (shift_change < 0)
134 tk->xtime_nsec >>= -shift_change;
135 else
136 tk->xtime_nsec <<= shift_change;
137 }
138 tk->shift = clock->shift;
139
140 tk->ntp_error = 0;
141 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
142
143
144
145
146
147
148 tk->mult = clock->mult;
149}
150
151
152
153#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
154u32 (*arch_gettimeoffset)(void);
155
156u32 get_arch_timeoffset(void)
157{
158 if (likely(arch_gettimeoffset))
159 return arch_gettimeoffset();
160 return 0;
161}
162#else
163static inline u32 get_arch_timeoffset(void) { return 0; }
164#endif
165
166static inline s64 timekeeping_get_ns(struct timekeeper *tk)
167{
168 cycle_t cycle_now, cycle_delta;
169 struct clocksource *clock;
170 s64 nsec;
171
172
173 clock = tk->clock;
174 cycle_now = clock->read(clock);
175
176
177 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
178
179 nsec = cycle_delta * tk->mult + tk->xtime_nsec;
180 nsec >>= tk->shift;
181
182
183 return nsec + get_arch_timeoffset();
184}
185
186static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
187{
188 cycle_t cycle_now, cycle_delta;
189 struct clocksource *clock;
190 s64 nsec;
191
192
193 clock = tk->clock;
194 cycle_now = clock->read(clock);
195
196
197 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
198
199
200 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
201
202
203 return nsec + get_arch_timeoffset();
204}
205
206static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
207
208static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
209{
210 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
211}
212
213
214
215
216int pvclock_gtod_register_notifier(struct notifier_block *nb)
217{
218 struct timekeeper *tk = &timekeeper;
219 unsigned long flags;
220 int ret;
221
222 raw_spin_lock_irqsave(&timekeeper_lock, flags);
223 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
224 update_pvclock_gtod(tk, true);
225 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
226
227 return ret;
228}
229EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
230
231
232
233
234
235int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
236{
237 unsigned long flags;
238 int ret;
239
240 raw_spin_lock_irqsave(&timekeeper_lock, flags);
241 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
242 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
243
244 return ret;
245}
246EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
247
248
249static void timekeeping_update(struct timekeeper *tk, unsigned int action)
250{
251 if (action & TK_CLEAR_NTP) {
252 tk->ntp_error = 0;
253 ntp_clear();
254 }
255 update_vsyscall(tk);
256 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
257
258 if (action & TK_MIRROR)
259 memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));
260}
261
262
263
264
265
266
267
268
269static void timekeeping_forward_now(struct timekeeper *tk)
270{
271 cycle_t cycle_now, cycle_delta;
272 struct clocksource *clock;
273 s64 nsec;
274
275 clock = tk->clock;
276 cycle_now = clock->read(clock);
277 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
278 tk->cycle_last = clock->cycle_last = cycle_now;
279
280 tk->xtime_nsec += cycle_delta * tk->mult;
281
282
283 tk->xtime_nsec += (u64)get_arch_timeoffset() << tk->shift;
284
285 tk_normalize_xtime(tk);
286
287 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
288 timespec_add_ns(&tk->raw_time, nsec);
289}
290
291
292
293
294
295
296
297
298int __getnstimeofday(struct timespec *ts)
299{
300 struct timekeeper *tk = &timekeeper;
301 unsigned long seq;
302 s64 nsecs = 0;
303
304 do {
305 seq = read_seqcount_begin(&timekeeper_seq);
306
307 ts->tv_sec = tk->xtime_sec;
308 nsecs = timekeeping_get_ns(tk);
309
310 } while (read_seqcount_retry(&timekeeper_seq, seq));
311
312 ts->tv_nsec = 0;
313 timespec_add_ns(ts, nsecs);
314
315
316
317
318
319 if (unlikely(timekeeping_suspended))
320 return -EAGAIN;
321 return 0;
322}
323EXPORT_SYMBOL(__getnstimeofday);
324
325
326
327
328
329
330
331void getnstimeofday(struct timespec *ts)
332{
333 WARN_ON(__getnstimeofday(ts));
334}
335EXPORT_SYMBOL(getnstimeofday);
336
337ktime_t ktime_get(void)
338{
339 struct timekeeper *tk = &timekeeper;
340 unsigned int seq;
341 s64 secs, nsecs;
342
343 WARN_ON(timekeeping_suspended);
344
345 do {
346 seq = read_seqcount_begin(&timekeeper_seq);
347 secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
348 nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;
349
350 } while (read_seqcount_retry(&timekeeper_seq, seq));
351
352
353
354
355 return ktime_add_ns(ktime_set(secs, 0), nsecs);
356}
357EXPORT_SYMBOL_GPL(ktime_get);
358
359
360
361
362
363
364
365
366
367void ktime_get_ts(struct timespec *ts)
368{
369 struct timekeeper *tk = &timekeeper;
370 struct timespec tomono;
371 s64 nsec;
372 unsigned int seq;
373
374 WARN_ON(timekeeping_suspended);
375
376 do {
377 seq = read_seqcount_begin(&timekeeper_seq);
378 ts->tv_sec = tk->xtime_sec;
379 nsec = timekeeping_get_ns(tk);
380 tomono = tk->wall_to_monotonic;
381
382 } while (read_seqcount_retry(&timekeeper_seq, seq));
383
384 ts->tv_sec += tomono.tv_sec;
385 ts->tv_nsec = 0;
386 timespec_add_ns(ts, nsec + tomono.tv_nsec);
387}
388EXPORT_SYMBOL_GPL(ktime_get_ts);
389
390
391
392
393
394
395
396
397void timekeeping_clocktai(struct timespec *ts)
398{
399 struct timekeeper *tk = &timekeeper;
400 unsigned long seq;
401 u64 nsecs;
402
403 WARN_ON(timekeeping_suspended);
404
405 do {
406 seq = read_seqcount_begin(&timekeeper_seq);
407
408 ts->tv_sec = tk->xtime_sec + tk->tai_offset;
409 nsecs = timekeeping_get_ns(tk);
410
411 } while (read_seqcount_retry(&timekeeper_seq, seq));
412
413 ts->tv_nsec = 0;
414 timespec_add_ns(ts, nsecs);
415
416}
417EXPORT_SYMBOL(timekeeping_clocktai);
418
419
420
421
422
423
424
425ktime_t ktime_get_clocktai(void)
426{
427 struct timespec ts;
428
429 timekeeping_clocktai(&ts);
430 return timespec_to_ktime(ts);
431}
432EXPORT_SYMBOL(ktime_get_clocktai);
433
434#ifdef CONFIG_NTP_PPS
435
436
437
438
439
440
441
442
443
444
445void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
446{
447 struct timekeeper *tk = &timekeeper;
448 unsigned long seq;
449 s64 nsecs_raw, nsecs_real;
450
451 WARN_ON_ONCE(timekeeping_suspended);
452
453 do {
454 seq = read_seqcount_begin(&timekeeper_seq);
455
456 *ts_raw = tk->raw_time;
457 ts_real->tv_sec = tk->xtime_sec;
458 ts_real->tv_nsec = 0;
459
460 nsecs_raw = timekeeping_get_ns_raw(tk);
461 nsecs_real = timekeeping_get_ns(tk);
462
463 } while (read_seqcount_retry(&timekeeper_seq, seq));
464
465 timespec_add_ns(ts_raw, nsecs_raw);
466 timespec_add_ns(ts_real, nsecs_real);
467}
468EXPORT_SYMBOL(getnstime_raw_and_real);
469
470#endif
471
472
473
474
475
476
477
478void do_gettimeofday(struct timeval *tv)
479{
480 struct timespec now;
481
482 getnstimeofday(&now);
483 tv->tv_sec = now.tv_sec;
484 tv->tv_usec = now.tv_nsec/1000;
485}
486EXPORT_SYMBOL(do_gettimeofday);
487
488
489
490
491
492
493
494int do_settimeofday(const struct timespec *tv)
495{
496 struct timekeeper *tk = &timekeeper;
497 struct timespec ts_delta, xt;
498 unsigned long flags;
499
500 if (!timespec_valid_strict(tv))
501 return -EINVAL;
502
503 raw_spin_lock_irqsave(&timekeeper_lock, flags);
504 write_seqcount_begin(&timekeeper_seq);
505
506 timekeeping_forward_now(tk);
507
508 xt = tk_xtime(tk);
509 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
510 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;
511
512 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta));
513
514 tk_set_xtime(tk, tv);
515
516 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
517
518 write_seqcount_end(&timekeeper_seq);
519 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
520
521
522 clock_was_set();
523
524 return 0;
525}
526EXPORT_SYMBOL(do_settimeofday);
527
528
529
530
531
532
533
534int timekeeping_inject_offset(struct timespec *ts)
535{
536 struct timekeeper *tk = &timekeeper;
537 unsigned long flags;
538 struct timespec tmp;
539 int ret = 0;
540
541 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
542 return -EINVAL;
543
544 raw_spin_lock_irqsave(&timekeeper_lock, flags);
545 write_seqcount_begin(&timekeeper_seq);
546
547 timekeeping_forward_now(tk);
548
549
550 tmp = timespec_add(tk_xtime(tk), *ts);
551 if (!timespec_valid_strict(&tmp)) {
552 ret = -EINVAL;
553 goto error;
554 }
555
556 tk_xtime_add(tk, ts);
557 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts));
558
559error:
560 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
561
562 write_seqcount_end(&timekeeper_seq);
563 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
564
565
566 clock_was_set();
567
568 return ret;
569}
570EXPORT_SYMBOL(timekeeping_inject_offset);
571
572
573
574
575
576
577s32 timekeeping_get_tai_offset(void)
578{
579 struct timekeeper *tk = &timekeeper;
580 unsigned int seq;
581 s32 ret;
582
583 do {
584 seq = read_seqcount_begin(&timekeeper_seq);
585 ret = tk->tai_offset;
586 } while (read_seqcount_retry(&timekeeper_seq, seq));
587
588 return ret;
589}
590
591
592
593
594
595static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
596{
597 tk->tai_offset = tai_offset;
598 tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tai_offset, 0));
599}
600
601
602
603
604
605void timekeeping_set_tai_offset(s32 tai_offset)
606{
607 struct timekeeper *tk = &timekeeper;
608 unsigned long flags;
609
610 raw_spin_lock_irqsave(&timekeeper_lock, flags);
611 write_seqcount_begin(&timekeeper_seq);
612 __timekeeping_set_tai_offset(tk, tai_offset);
613 write_seqcount_end(&timekeeper_seq);
614 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
615 clock_was_set();
616}
617
618
619
620
621
622
623static int change_clocksource(void *data)
624{
625 struct timekeeper *tk = &timekeeper;
626 struct clocksource *new, *old;
627 unsigned long flags;
628
629 new = (struct clocksource *) data;
630
631 raw_spin_lock_irqsave(&timekeeper_lock, flags);
632 write_seqcount_begin(&timekeeper_seq);
633
634 timekeeping_forward_now(tk);
635
636
637
638
639 if (try_module_get(new->owner)) {
640 if (!new->enable || new->enable(new) == 0) {
641 old = tk->clock;
642 tk_setup_internals(tk, new);
643 if (old->disable)
644 old->disable(old);
645 module_put(old->owner);
646 } else {
647 module_put(new->owner);
648 }
649 }
650 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
651
652 write_seqcount_end(&timekeeper_seq);
653 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
654
655 return 0;
656}
657
658
659
660
661
662
663
664
665int timekeeping_notify(struct clocksource *clock)
666{
667 struct timekeeper *tk = &timekeeper;
668
669 if (tk->clock == clock)
670 return 0;
671 stop_machine(change_clocksource, clock, NULL);
672 tick_clock_notify();
673 return tk->clock == clock ? 0 : -1;
674}
675
676
677
678
679
680
681ktime_t ktime_get_real(void)
682{
683 struct timespec now;
684
685 getnstimeofday(&now);
686
687 return timespec_to_ktime(now);
688}
689EXPORT_SYMBOL_GPL(ktime_get_real);
690
691
692
693
694
695
696
697void getrawmonotonic(struct timespec *ts)
698{
699 struct timekeeper *tk = &timekeeper;
700 unsigned long seq;
701 s64 nsecs;
702
703 do {
704 seq = read_seqcount_begin(&timekeeper_seq);
705 nsecs = timekeeping_get_ns_raw(tk);
706 *ts = tk->raw_time;
707
708 } while (read_seqcount_retry(&timekeeper_seq, seq));
709
710 timespec_add_ns(ts, nsecs);
711}
712EXPORT_SYMBOL(getrawmonotonic);
713
714
715
716
717int timekeeping_valid_for_hres(void)
718{
719 struct timekeeper *tk = &timekeeper;
720 unsigned long seq;
721 int ret;
722
723 do {
724 seq = read_seqcount_begin(&timekeeper_seq);
725
726 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
727
728 } while (read_seqcount_retry(&timekeeper_seq, seq));
729
730 return ret;
731}
732
733
734
735
736u64 timekeeping_max_deferment(void)
737{
738 struct timekeeper *tk = &timekeeper;
739 unsigned long seq;
740 u64 ret;
741
742 do {
743 seq = read_seqcount_begin(&timekeeper_seq);
744
745 ret = tk->clock->max_idle_ns;
746
747 } while (read_seqcount_retry(&timekeeper_seq, seq));
748
749 return ret;
750}
751
752
753
754
755
756
757
758
759
760
761void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
762{
763 ts->tv_sec = 0;
764 ts->tv_nsec = 0;
765}
766
767
768
769
770
771
772
773
774
775
776void __attribute__((weak)) read_boot_clock(struct timespec *ts)
777{
778 ts->tv_sec = 0;
779 ts->tv_nsec = 0;
780}
781
782
783
784
785void __init timekeeping_init(void)
786{
787 struct timekeeper *tk = &timekeeper;
788 struct clocksource *clock;
789 unsigned long flags;
790 struct timespec now, boot, tmp;
791
792 read_persistent_clock(&now);
793
794 if (!timespec_valid_strict(&now)) {
795 pr_warn("WARNING: Persistent clock returned invalid value!\n"
796 " Check your CMOS/BIOS settings.\n");
797 now.tv_sec = 0;
798 now.tv_nsec = 0;
799 } else if (now.tv_sec || now.tv_nsec)
800 persistent_clock_exist = true;
801
802 read_boot_clock(&boot);
803 if (!timespec_valid_strict(&boot)) {
804 pr_warn("WARNING: Boot clock returned invalid value!\n"
805 " Check your CMOS/BIOS settings.\n");
806 boot.tv_sec = 0;
807 boot.tv_nsec = 0;
808 }
809
810 raw_spin_lock_irqsave(&timekeeper_lock, flags);
811 write_seqcount_begin(&timekeeper_seq);
812 ntp_init();
813
814 clock = clocksource_default_clock();
815 if (clock->enable)
816 clock->enable(clock);
817 tk_setup_internals(tk, clock);
818
819 tk_set_xtime(tk, &now);
820 tk->raw_time.tv_sec = 0;
821 tk->raw_time.tv_nsec = 0;
822 if (boot.tv_sec == 0 && boot.tv_nsec == 0)
823 boot = tk_xtime(tk);
824
825 set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec);
826 tk_set_wall_to_mono(tk, tmp);
827
828 tmp.tv_sec = 0;
829 tmp.tv_nsec = 0;
830 tk_set_sleep_time(tk, tmp);
831
832 memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));
833
834 write_seqcount_end(&timekeeper_seq);
835 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
836}
837
838
839static struct timespec timekeeping_suspend_time;
840
841
842
843
844
845
846
847
848static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
849 struct timespec *delta)
850{
851 if (!timespec_valid_strict(delta)) {
852 printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
853 "sleep delta value!\n");
854 return;
855 }
856 tk_xtime_add(tk, delta);
857 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta));
858 tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta));
859 tk_debug_account_sleep_time(delta);
860}
861
862
863
864
865
866
867
868
869
870
871
872void timekeeping_inject_sleeptime(struct timespec *delta)
873{
874 struct timekeeper *tk = &timekeeper;
875 unsigned long flags;
876
877
878
879
880
881 if (has_persistent_clock())
882 return;
883
884 raw_spin_lock_irqsave(&timekeeper_lock, flags);
885 write_seqcount_begin(&timekeeper_seq);
886
887 timekeeping_forward_now(tk);
888
889 __timekeeping_inject_sleeptime(tk, delta);
890
891 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
892
893 write_seqcount_end(&timekeeper_seq);
894 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
895
896
897 clock_was_set();
898}
899
900
901
902
903
904
905
906
907static void timekeeping_resume(void)
908{
909 struct timekeeper *tk = &timekeeper;
910 struct clocksource *clock = tk->clock;
911 unsigned long flags;
912 struct timespec ts_new, ts_delta;
913 cycle_t cycle_now, cycle_delta;
914 bool suspendtime_found = false;
915
916 read_persistent_clock(&ts_new);
917
918 clockevents_resume();
919 clocksource_resume();
920
921 raw_spin_lock_irqsave(&timekeeper_lock, flags);
922 write_seqcount_begin(&timekeeper_seq);
923
924
925
926
927
928
929
930
931
932
933
934
935
936 cycle_now = clock->read(clock);
937 if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
938 cycle_now > clock->cycle_last) {
939 u64 num, max = ULLONG_MAX;
940 u32 mult = clock->mult;
941 u32 shift = clock->shift;
942 s64 nsec = 0;
943
944 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
945
946
947
948
949
950
951 do_div(max, mult);
952 if (cycle_delta > max) {
953 num = div64_u64(cycle_delta, max);
954 nsec = (((u64) max * mult) >> shift) * num;
955 cycle_delta -= num * max;
956 }
957 nsec += ((u64) cycle_delta * mult) >> shift;
958
959 ts_delta = ns_to_timespec(nsec);
960 suspendtime_found = true;
961 } else if (timespec_compare(&ts_new, &timekeeping_suspend_time) > 0) {
962 ts_delta = timespec_sub(ts_new, timekeeping_suspend_time);
963 suspendtime_found = true;
964 }
965
966 if (suspendtime_found)
967 __timekeeping_inject_sleeptime(tk, &ts_delta);
968
969
970 tk->cycle_last = clock->cycle_last = cycle_now;
971 tk->ntp_error = 0;
972 timekeeping_suspended = 0;
973 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
974 write_seqcount_end(&timekeeper_seq);
975 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
976
977 touch_softlockup_watchdog();
978
979 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
980
981
982 hrtimers_resume();
983}
984
985static int timekeeping_suspend(void)
986{
987 struct timekeeper *tk = &timekeeper;
988 unsigned long flags;
989 struct timespec delta, delta_delta;
990 static struct timespec old_delta;
991
992 read_persistent_clock(&timekeeping_suspend_time);
993
994
995
996
997
998
999 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1000 persistent_clock_exist = true;
1001
1002 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1003 write_seqcount_begin(&timekeeper_seq);
1004 timekeeping_forward_now(tk);
1005 timekeeping_suspended = 1;
1006
1007
1008
1009
1010
1011
1012
1013 delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time);
1014 delta_delta = timespec_sub(delta, old_delta);
1015 if (abs(delta_delta.tv_sec) >= 2) {
1016
1017
1018
1019
1020 old_delta = delta;
1021 } else {
1022
1023 timekeeping_suspend_time =
1024 timespec_add(timekeeping_suspend_time, delta_delta);
1025 }
1026 write_seqcount_end(&timekeeper_seq);
1027 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1028
1029 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
1030 clocksource_suspend();
1031 clockevents_suspend();
1032
1033 return 0;
1034}
1035
1036
1037static struct syscore_ops timekeeping_syscore_ops = {
1038 .resume = timekeeping_resume,
1039 .suspend = timekeeping_suspend,
1040};
1041
1042static int __init timekeeping_init_ops(void)
1043{
1044 register_syscore_ops(&timekeeping_syscore_ops);
1045 return 0;
1046}
1047
1048device_initcall(timekeeping_init_ops);
1049
1050
1051
1052
1053
1054static __always_inline int timekeeping_bigadjust(struct timekeeper *tk,
1055 s64 error, s64 *interval,
1056 s64 *offset)
1057{
1058 s64 tick_error, i;
1059 u32 look_ahead, adj;
1060 s32 error2, mult;
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071 error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
1072 error2 = abs(error2);
1073 for (look_ahead = 0; error2 > 0; look_ahead++)
1074 error2 >>= 2;
1075
1076
1077
1078
1079
1080 tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1);
1081 tick_error -= tk->xtime_interval >> 1;
1082 error = ((error - tick_error) >> look_ahead) + tick_error;
1083
1084
1085 i = *interval;
1086 mult = 1;
1087 if (error < 0) {
1088 error = -error;
1089 *interval = -*interval;
1090 *offset = -*offset;
1091 mult = -1;
1092 }
1093 for (adj = 0; error > i; adj++)
1094 error >>= 1;
1095
1096 *interval <<= adj;
1097 *offset <<= adj;
1098 return mult << adj;
1099}
1100
1101
1102
1103
1104
1105
1106static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1107{
1108 s64 error, interval = tk->cycle_interval;
1109 int adj;
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124 error = tk->ntp_error >> (tk->ntp_error_shift - 1);
1125 if (error > interval) {
1126
1127
1128
1129
1130
1131
1132 error >>= 2;
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143 if (likely(error <= interval))
1144 adj = 1;
1145 else
1146 adj = timekeeping_bigadjust(tk, error, &interval, &offset);
1147 } else {
1148 if (error < -interval) {
1149
1150 error >>= 2;
1151 if (likely(error >= -interval)) {
1152 adj = -1;
1153 interval = -interval;
1154 offset = -offset;
1155 } else {
1156 adj = timekeeping_bigadjust(tk, error, &interval, &offset);
1157 }
1158 } else {
1159 goto out_adjust;
1160 }
1161 }
1162
1163 if (unlikely(tk->clock->maxadj &&
1164 (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) {
1165 printk_once(KERN_WARNING
1166 "Adjusting %s more than 11%% (%ld vs %ld)\n",
1167 tk->clock->name, (long)tk->mult + adj,
1168 (long)tk->clock->mult + tk->clock->maxadj);
1169 }
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219 tk->mult += adj;
1220 tk->xtime_interval += interval;
1221 tk->xtime_nsec -= offset;
1222 tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
1223
1224out_adjust:
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239 if (unlikely((s64)tk->xtime_nsec < 0)) {
1240 s64 neg = -(s64)tk->xtime_nsec;
1241 tk->xtime_nsec = 0;
1242 tk->ntp_error += neg << tk->ntp_error_shift;
1243 }
1244
1245}
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1256{
1257 u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
1258 unsigned int action = 0;
1259
1260 while (tk->xtime_nsec >= nsecps) {
1261 int leap;
1262
1263 tk->xtime_nsec -= nsecps;
1264 tk->xtime_sec++;
1265
1266
1267 leap = second_overflow(tk->xtime_sec);
1268 if (unlikely(leap)) {
1269 struct timespec ts;
1270
1271 tk->xtime_sec += leap;
1272
1273 ts.tv_sec = leap;
1274 ts.tv_nsec = 0;
1275 tk_set_wall_to_mono(tk,
1276 timespec_sub(tk->wall_to_monotonic, ts));
1277
1278 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1279
1280 clock_was_set_delayed();
1281 action = TK_CLOCK_WAS_SET;
1282 }
1283 }
1284 return action;
1285}
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1297 u32 shift)
1298{
1299 cycle_t interval = tk->cycle_interval << shift;
1300 u64 raw_nsecs;
1301
1302
1303 if (offset < interval)
1304 return offset;
1305
1306
1307 offset -= interval;
1308 tk->cycle_last += interval;
1309
1310 tk->xtime_nsec += tk->xtime_interval << shift;
1311 accumulate_nsecs_to_secs(tk);
1312
1313
1314 raw_nsecs = (u64)tk->raw_interval << shift;
1315 raw_nsecs += tk->raw_time.tv_nsec;
1316 if (raw_nsecs >= NSEC_PER_SEC) {
1317 u64 raw_secs = raw_nsecs;
1318 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
1319 tk->raw_time.tv_sec += raw_secs;
1320 }
1321 tk->raw_time.tv_nsec = raw_nsecs;
1322
1323
1324 tk->ntp_error += ntp_tick_length() << shift;
1325 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
1326 (tk->ntp_error_shift + shift);
1327
1328 return offset;
1329}
1330
1331#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
1332static inline void old_vsyscall_fixup(struct timekeeper *tk)
1333{
1334 s64 remainder;
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346 remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
1347 tk->xtime_nsec -= remainder;
1348 tk->xtime_nsec += 1ULL << tk->shift;
1349 tk->ntp_error += remainder << tk->ntp_error_shift;
1350
1351}
1352#else
1353#define old_vsyscall_fixup(tk)
1354#endif
1355
1356
1357
1358
1359
1360
1361
1362static void update_wall_time(void)
1363{
1364 struct clocksource *clock;
1365 struct timekeeper *real_tk = &timekeeper;
1366 struct timekeeper *tk = &shadow_timekeeper;
1367 cycle_t offset;
1368 int shift = 0, maxshift;
1369 unsigned int action;
1370 unsigned long flags;
1371
1372 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1373
1374
1375 if (unlikely(timekeeping_suspended))
1376 goto out;
1377
1378 clock = real_tk->clock;
1379
1380#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
1381 offset = real_tk->cycle_interval;
1382#else
1383 offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
1384#endif
1385
1386
1387 if (offset < real_tk->cycle_interval)
1388 goto out;
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398 shift = ilog2(offset) - ilog2(tk->cycle_interval);
1399 shift = max(0, shift);
1400
1401 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
1402 shift = min(shift, maxshift);
1403 while (offset >= tk->cycle_interval) {
1404 offset = logarithmic_accumulation(tk, offset, shift);
1405 if (offset < tk->cycle_interval<<shift)
1406 shift--;
1407 }
1408
1409
1410 timekeeping_adjust(tk, offset);
1411
1412
1413
1414
1415
1416 old_vsyscall_fixup(tk);
1417
1418
1419
1420
1421
1422 action = accumulate_nsecs_to_secs(tk);
1423
1424 write_seqcount_begin(&timekeeper_seq);
1425
1426 clock->cycle_last = tk->cycle_last;
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437 memcpy(real_tk, tk, sizeof(*tk));
1438 timekeeping_update(real_tk, action);
1439 write_seqcount_end(&timekeeper_seq);
1440out:
1441 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1442}
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455void getboottime(struct timespec *ts)
1456{
1457 struct timekeeper *tk = &timekeeper;
1458 struct timespec boottime = {
1459 .tv_sec = tk->wall_to_monotonic.tv_sec +
1460 tk->total_sleep_time.tv_sec,
1461 .tv_nsec = tk->wall_to_monotonic.tv_nsec +
1462 tk->total_sleep_time.tv_nsec
1463 };
1464
1465 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
1466}
1467EXPORT_SYMBOL_GPL(getboottime);
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478void get_monotonic_boottime(struct timespec *ts)
1479{
1480 struct timekeeper *tk = &timekeeper;
1481 struct timespec tomono, sleep;
1482 s64 nsec;
1483 unsigned int seq;
1484
1485 WARN_ON(timekeeping_suspended);
1486
1487 do {
1488 seq = read_seqcount_begin(&timekeeper_seq);
1489 ts->tv_sec = tk->xtime_sec;
1490 nsec = timekeeping_get_ns(tk);
1491 tomono = tk->wall_to_monotonic;
1492 sleep = tk->total_sleep_time;
1493
1494 } while (read_seqcount_retry(&timekeeper_seq, seq));
1495
1496 ts->tv_sec += tomono.tv_sec + sleep.tv_sec;
1497 ts->tv_nsec = 0;
1498 timespec_add_ns(ts, nsec + tomono.tv_nsec + sleep.tv_nsec);
1499}
1500EXPORT_SYMBOL_GPL(get_monotonic_boottime);
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510ktime_t ktime_get_boottime(void)
1511{
1512 struct timespec ts;
1513
1514 get_monotonic_boottime(&ts);
1515 return timespec_to_ktime(ts);
1516}
1517EXPORT_SYMBOL_GPL(ktime_get_boottime);
1518
1519
1520
1521
1522
1523void monotonic_to_bootbased(struct timespec *ts)
1524{
1525 struct timekeeper *tk = &timekeeper;
1526
1527 *ts = timespec_add(*ts, tk->total_sleep_time);
1528}
1529EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
1530
1531unsigned long get_seconds(void)
1532{
1533 struct timekeeper *tk = &timekeeper;
1534
1535 return tk->xtime_sec;
1536}
1537EXPORT_SYMBOL(get_seconds);
1538
1539struct timespec __current_kernel_time(void)
1540{
1541 struct timekeeper *tk = &timekeeper;
1542
1543 return tk_xtime(tk);
1544}
1545
1546struct timespec current_kernel_time(void)
1547{
1548 struct timekeeper *tk = &timekeeper;
1549 struct timespec now;
1550 unsigned long seq;
1551
1552 do {
1553 seq = read_seqcount_begin(&timekeeper_seq);
1554
1555 now = tk_xtime(tk);
1556 } while (read_seqcount_retry(&timekeeper_seq, seq));
1557
1558 return now;
1559}
1560EXPORT_SYMBOL(current_kernel_time);
1561
1562struct timespec get_monotonic_coarse(void)
1563{
1564 struct timekeeper *tk = &timekeeper;
1565 struct timespec now, mono;
1566 unsigned long seq;
1567
1568 do {
1569 seq = read_seqcount_begin(&timekeeper_seq);
1570
1571 now = tk_xtime(tk);
1572 mono = tk->wall_to_monotonic;
1573 } while (read_seqcount_retry(&timekeeper_seq, seq));
1574
1575 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
1576 now.tv_nsec + mono.tv_nsec);
1577 return now;
1578}
1579
1580
1581
1582
1583void do_timer(unsigned long ticks)
1584{
1585 jiffies_64 += ticks;
1586 update_wall_time();
1587 calc_global_load(ticks);
1588}
1589
1590
1591
1592
1593
1594
1595
1596
1597void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1598 struct timespec *wtom, struct timespec *sleep)
1599{
1600 struct timekeeper *tk = &timekeeper;
1601 unsigned long seq;
1602
1603 do {
1604 seq = read_seqcount_begin(&timekeeper_seq);
1605 *xtim = tk_xtime(tk);
1606 *wtom = tk->wall_to_monotonic;
1607 *sleep = tk->total_sleep_time;
1608 } while (read_seqcount_retry(&timekeeper_seq, seq));
1609}
1610
1611#ifdef CONFIG_HIGH_RES_TIMERS
1612
1613
1614
1615
1616
1617
1618
1619
1620ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot,
1621 ktime_t *offs_tai)
1622{
1623 struct timekeeper *tk = &timekeeper;
1624 ktime_t now;
1625 unsigned int seq;
1626 u64 secs, nsecs;
1627
1628 do {
1629 seq = read_seqcount_begin(&timekeeper_seq);
1630
1631 secs = tk->xtime_sec;
1632 nsecs = timekeeping_get_ns(tk);
1633
1634 *offs_real = tk->offs_real;
1635 *offs_boot = tk->offs_boot;
1636 *offs_tai = tk->offs_tai;
1637 } while (read_seqcount_retry(&timekeeper_seq, seq));
1638
1639 now = ktime_add_ns(ktime_set(secs, 0), nsecs);
1640 now = ktime_sub(now, *offs_real);
1641 return now;
1642}
1643#endif
1644
1645
1646
1647
1648ktime_t ktime_get_monotonic_offset(void)
1649{
1650 struct timekeeper *tk = &timekeeper;
1651 unsigned long seq;
1652 struct timespec wtom;
1653
1654 do {
1655 seq = read_seqcount_begin(&timekeeper_seq);
1656 wtom = tk->wall_to_monotonic;
1657 } while (read_seqcount_retry(&timekeeper_seq, seq));
1658
1659 return timespec_to_ktime(wtom);
1660}
1661EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
1662
1663
1664
1665
1666int do_adjtimex(struct timex *txc)
1667{
1668 struct timekeeper *tk = &timekeeper;
1669 unsigned long flags;
1670 struct timespec ts;
1671 s32 orig_tai, tai;
1672 int ret;
1673
1674
1675 ret = ntp_validate_timex(txc);
1676 if (ret)
1677 return ret;
1678
1679 if (txc->modes & ADJ_SETOFFSET) {
1680 struct timespec delta;
1681 delta.tv_sec = txc->time.tv_sec;
1682 delta.tv_nsec = txc->time.tv_usec;
1683 if (!(txc->modes & ADJ_NANO))
1684 delta.tv_nsec *= 1000;
1685 ret = timekeeping_inject_offset(&delta);
1686 if (ret)
1687 return ret;
1688 }
1689
1690 getnstimeofday(&ts);
1691
1692 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1693 write_seqcount_begin(&timekeeper_seq);
1694
1695 orig_tai = tai = tk->tai_offset;
1696 ret = __do_adjtimex(txc, &ts, &tai);
1697
1698 if (tai != orig_tai) {
1699 __timekeeping_set_tai_offset(tk, tai);
1700 update_pvclock_gtod(tk, true);
1701 clock_was_set_delayed();
1702 }
1703 write_seqcount_end(&timekeeper_seq);
1704 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1705
1706 return ret;
1707}
1708
1709#ifdef CONFIG_NTP_PPS
1710
1711
1712
1713void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts)
1714{
1715 unsigned long flags;
1716
1717 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1718 write_seqcount_begin(&timekeeper_seq);
1719
1720 __hardpps(phase_ts, raw_ts);
1721
1722 write_seqcount_end(&timekeeper_seq);
1723 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1724}
1725EXPORT_SYMBOL(hardpps);
1726#endif
1727
1728
1729
1730
1731
1732
1733
1734void xtime_update(unsigned long ticks)
1735{
1736 write_seqlock(&jiffies_lock);
1737 do_timer(ticks);
1738 write_sequnlock(&jiffies_lock);
1739}
1740