1
2
3
4
5
6
7
8
9
10
11#include <linux/timekeeper_internal.h>
12#include <linux/module.h>
13#include <linux/interrupt.h>
14#include <linux/percpu.h>
15#include <linux/init.h>
16#include <linux/mm.h>
17#include <linux/sched.h>
18#include <linux/syscore_ops.h>
19#include <linux/clocksource.h>
20#include <linux/jiffies.h>
21#include <linux/time.h>
22#include <linux/tick.h>
23#include <linux/stop_machine.h>
24#include <linux/pvclock_gtod.h>
25#include <linux/compiler.h>
26
27#include "tick-internal.h"
28#include "ntp_internal.h"
29#include "timekeeping_internal.h"
30
31#define TK_CLEAR_NTP (1 << 0)
32#define TK_MIRROR (1 << 1)
33#define TK_CLOCK_WAS_SET (1 << 2)
34
35
36
37
38
39static struct {
40 seqcount_t seq;
41 struct timekeeper timekeeper;
42} tk_core ____cacheline_aligned;
43
44static DEFINE_RAW_SPINLOCK(timekeeper_lock);
45static struct timekeeper shadow_timekeeper;
46
47
48
49
50
51
52
53
54
55
56struct tk_fast {
57 seqcount_t seq;
58 struct tk_read_base base[2];
59};
60
61static struct tk_fast tk_fast_mono ____cacheline_aligned;
62static struct tk_fast tk_fast_raw ____cacheline_aligned;
63
64
65int __read_mostly timekeeping_suspended;
66
67static inline void tk_normalize_xtime(struct timekeeper *tk)
68{
69 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
70 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
71 tk->xtime_sec++;
72 }
73}
74
75static inline struct timespec64 tk_xtime(struct timekeeper *tk)
76{
77 struct timespec64 ts;
78
79 ts.tv_sec = tk->xtime_sec;
80 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
81 return ts;
82}
83
84static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
85{
86 tk->xtime_sec = ts->tv_sec;
87 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
88}
89
90static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
91{
92 tk->xtime_sec += ts->tv_sec;
93 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
94 tk_normalize_xtime(tk);
95}
96
97static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
98{
99 struct timespec64 tmp;
100
101
102
103
104
105 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
106 -tk->wall_to_monotonic.tv_nsec);
107 WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64);
108 tk->wall_to_monotonic = wtm;
109 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
110 tk->offs_real = timespec64_to_ktime(tmp);
111 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
112}
113
114static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
115{
116 tk->offs_boot = ktime_add(tk->offs_boot, delta);
117}
118
119#ifdef CONFIG_DEBUG_TIMEKEEPING
120#define WARNING_FREQ (HZ*300)
121
122static void timekeeping_check_update(struct timekeeper *tk, cycle_t offset)
123{
124
125 cycle_t max_cycles = tk->tkr_mono.clock->max_cycles;
126 const char *name = tk->tkr_mono.clock->name;
127
128 if (offset > max_cycles) {
129 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
130 offset, name, max_cycles);
131 printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
132 } else {
133 if (offset > (max_cycles >> 1)) {
134 printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
135 offset, name, max_cycles >> 1);
136 printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
137 }
138 }
139
140 if (tk->underflow_seen) {
141 if (jiffies - tk->last_warning > WARNING_FREQ) {
142 printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
143 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
144 printk_deferred(" Your kernel is probably still fine.\n");
145 tk->last_warning = jiffies;
146 }
147 tk->underflow_seen = 0;
148 }
149
150 if (tk->overflow_seen) {
151 if (jiffies - tk->last_warning > WARNING_FREQ) {
152 printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
153 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
154 printk_deferred(" Your kernel is probably still fine.\n");
155 tk->last_warning = jiffies;
156 }
157 tk->overflow_seen = 0;
158 }
159}
160
161static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr)
162{
163 struct timekeeper *tk = &tk_core.timekeeper;
164 cycle_t now, last, mask, max, delta;
165 unsigned int seq;
166
167
168
169
170
171
172
173
174 do {
175 seq = read_seqcount_begin(&tk_core.seq);
176 now = tkr->read(tkr->clock);
177 last = tkr->cycle_last;
178 mask = tkr->mask;
179 max = tkr->clock->max_cycles;
180 } while (read_seqcount_retry(&tk_core.seq, seq));
181
182 delta = clocksource_delta(now, last, mask);
183
184
185
186
187
188 if (unlikely((~delta & mask) < (mask >> 3))) {
189 tk->underflow_seen = 1;
190 delta = 0;
191 }
192
193
194 if (unlikely(delta > max)) {
195 tk->overflow_seen = 1;
196 delta = tkr->clock->max_cycles;
197 }
198
199 return delta;
200}
201#else
202static inline void timekeeping_check_update(struct timekeeper *tk, cycle_t offset)
203{
204}
205static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr)
206{
207 cycle_t cycle_now, delta;
208
209
210 cycle_now = tkr->read(tkr->clock);
211
212
213 delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
214
215 return delta;
216}
217#endif
218
219
220
221
222
223
224
225
226
227
228
229
230static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
231{
232 cycle_t interval;
233 u64 tmp, ntpinterval;
234 struct clocksource *old_clock;
235
236 ++tk->cs_was_changed_seq;
237 old_clock = tk->tkr_mono.clock;
238 tk->tkr_mono.clock = clock;
239 tk->tkr_mono.read = clock->read;
240 tk->tkr_mono.mask = clock->mask;
241 tk->tkr_mono.cycle_last = tk->tkr_mono.read(clock);
242
243 tk->tkr_raw.clock = clock;
244 tk->tkr_raw.read = clock->read;
245 tk->tkr_raw.mask = clock->mask;
246 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
247
248
249 tmp = NTP_INTERVAL_LENGTH;
250 tmp <<= clock->shift;
251 ntpinterval = tmp;
252 tmp += clock->mult/2;
253 do_div(tmp, clock->mult);
254 if (tmp == 0)
255 tmp = 1;
256
257 interval = (cycle_t) tmp;
258 tk->cycle_interval = interval;
259
260
261 tk->xtime_interval = (u64) interval * clock->mult;
262 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
263 tk->raw_interval =
264 ((u64) interval * clock->mult) >> clock->shift;
265
266
267 if (old_clock) {
268 int shift_change = clock->shift - old_clock->shift;
269 if (shift_change < 0)
270 tk->tkr_mono.xtime_nsec >>= -shift_change;
271 else
272 tk->tkr_mono.xtime_nsec <<= shift_change;
273 }
274 tk->tkr_raw.xtime_nsec = 0;
275
276 tk->tkr_mono.shift = clock->shift;
277 tk->tkr_raw.shift = clock->shift;
278
279 tk->ntp_error = 0;
280 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
281 tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
282
283
284
285
286
287
288 tk->tkr_mono.mult = clock->mult;
289 tk->tkr_raw.mult = clock->mult;
290 tk->ntp_err_mult = 0;
291}
292
293
294
295#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
296static u32 default_arch_gettimeoffset(void) { return 0; }
297u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
298#else
299static inline u32 arch_gettimeoffset(void) { return 0; }
300#endif
301
302static inline s64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
303 cycle_t delta)
304{
305 s64 nsec;
306
307 nsec = delta * tkr->mult + tkr->xtime_nsec;
308 nsec >>= tkr->shift;
309
310
311 return nsec + arch_gettimeoffset();
312}
313
314static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
315{
316 cycle_t delta;
317
318 delta = timekeeping_get_delta(tkr);
319 return timekeeping_delta_to_ns(tkr, delta);
320}
321
322static inline s64 timekeeping_cycles_to_ns(struct tk_read_base *tkr,
323 cycle_t cycles)
324{
325 cycle_t delta;
326
327
328 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
329 return timekeeping_delta_to_ns(tkr, delta);
330}
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346static void update_fast_timekeeper(struct tk_read_base *tkr, struct tk_fast *tkf)
347{
348 struct tk_read_base *base = tkf->base;
349
350
351 raw_write_seqcount_latch(&tkf->seq);
352
353
354 memcpy(base, tkr, sizeof(*base));
355
356
357 raw_write_seqcount_latch(&tkf->seq);
358
359
360 memcpy(base + 1, base, sizeof(*base));
361}
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
396{
397 struct tk_read_base *tkr;
398 unsigned int seq;
399 u64 now;
400
401 do {
402 seq = raw_read_seqcount_latch(&tkf->seq);
403 tkr = tkf->base + (seq & 0x01);
404 now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr);
405 } while (read_seqcount_retry(&tkf->seq, seq));
406
407 return now;
408}
409
410u64 ktime_get_mono_fast_ns(void)
411{
412 return __ktime_get_fast_ns(&tk_fast_mono);
413}
414EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
415
416u64 ktime_get_raw_fast_ns(void)
417{
418 return __ktime_get_fast_ns(&tk_fast_raw);
419}
420EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
421
422
423static cycle_t cycles_at_suspend;
424
425static cycle_t dummy_clock_read(struct clocksource *cs)
426{
427 return cycles_at_suspend;
428}
429
430
431
432
433
434
435
436
437
438
439
440static void halt_fast_timekeeper(struct timekeeper *tk)
441{
442 static struct tk_read_base tkr_dummy;
443 struct tk_read_base *tkr = &tk->tkr_mono;
444
445 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
446 cycles_at_suspend = tkr->read(tkr->clock);
447 tkr_dummy.read = dummy_clock_read;
448 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
449
450 tkr = &tk->tkr_raw;
451 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
452 tkr_dummy.read = dummy_clock_read;
453 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
454}
455
456#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
457
458static inline void update_vsyscall(struct timekeeper *tk)
459{
460 struct timespec xt, wm;
461
462 xt = timespec64_to_timespec(tk_xtime(tk));
463 wm = timespec64_to_timespec(tk->wall_to_monotonic);
464 update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult,
465 tk->tkr_mono.cycle_last);
466}
467
468static inline void old_vsyscall_fixup(struct timekeeper *tk)
469{
470 s64 remainder;
471
472
473
474
475
476
477
478
479
480
481
482 remainder = tk->tkr_mono.xtime_nsec & ((1ULL << tk->tkr_mono.shift) - 1);
483 tk->tkr_mono.xtime_nsec -= remainder;
484 tk->tkr_mono.xtime_nsec += 1ULL << tk->tkr_mono.shift;
485 tk->ntp_error += remainder << tk->ntp_error_shift;
486 tk->ntp_error -= (1ULL << tk->tkr_mono.shift) << tk->ntp_error_shift;
487}
488#else
489#define old_vsyscall_fixup(tk)
490#endif
491
492static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
493
494static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
495{
496 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
497}
498
499
500
501
502int pvclock_gtod_register_notifier(struct notifier_block *nb)
503{
504 struct timekeeper *tk = &tk_core.timekeeper;
505 unsigned long flags;
506 int ret;
507
508 raw_spin_lock_irqsave(&timekeeper_lock, flags);
509 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
510 update_pvclock_gtod(tk, true);
511 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
512
513 return ret;
514}
515EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
516
517
518
519
520
521int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
522{
523 unsigned long flags;
524 int ret;
525
526 raw_spin_lock_irqsave(&timekeeper_lock, flags);
527 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
528 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
529
530 return ret;
531}
532EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
533
534
535
536
537static inline void tk_update_leap_state(struct timekeeper *tk)
538{
539 tk->next_leap_ktime = ntp_get_next_leap();
540 if (tk->next_leap_ktime.tv64 != KTIME_MAX)
541
542 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
543}
544
545
546
547
548static inline void tk_update_ktime_data(struct timekeeper *tk)
549{
550 u64 seconds;
551 u32 nsec;
552
553
554
555
556
557
558
559
560 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
561 nsec = (u32) tk->wall_to_monotonic.tv_nsec;
562 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
563
564
565 tk->tkr_raw.base = timespec64_to_ktime(tk->raw_time);
566
567
568
569
570
571
572 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
573 if (nsec >= NSEC_PER_SEC)
574 seconds++;
575 tk->ktime_sec = seconds;
576}
577
578
579static void timekeeping_update(struct timekeeper *tk, unsigned int action)
580{
581 if (action & TK_CLEAR_NTP) {
582 tk->ntp_error = 0;
583 ntp_clear();
584 }
585
586 tk_update_leap_state(tk);
587 tk_update_ktime_data(tk);
588
589 update_vsyscall(tk);
590 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
591
592 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
593 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
594
595 if (action & TK_CLOCK_WAS_SET)
596 tk->clock_was_set_seq++;
597
598
599
600
601
602 if (action & TK_MIRROR)
603 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
604 sizeof(tk_core.timekeeper));
605}
606
607
608
609
610
611
612
613
614static void timekeeping_forward_now(struct timekeeper *tk)
615{
616 struct clocksource *clock = tk->tkr_mono.clock;
617 cycle_t cycle_now, delta;
618 s64 nsec;
619
620 cycle_now = tk->tkr_mono.read(clock);
621 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
622 tk->tkr_mono.cycle_last = cycle_now;
623 tk->tkr_raw.cycle_last = cycle_now;
624
625 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
626
627
628 tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift;
629
630 tk_normalize_xtime(tk);
631
632 nsec = clocksource_cyc2ns(delta, tk->tkr_raw.mult, tk->tkr_raw.shift);
633 timespec64_add_ns(&tk->raw_time, nsec);
634}
635
636
637
638
639
640
641
642
643int __getnstimeofday64(struct timespec64 *ts)
644{
645 struct timekeeper *tk = &tk_core.timekeeper;
646 unsigned long seq;
647 s64 nsecs = 0;
648
649 do {
650 seq = read_seqcount_begin(&tk_core.seq);
651
652 ts->tv_sec = tk->xtime_sec;
653 nsecs = timekeeping_get_ns(&tk->tkr_mono);
654
655 } while (read_seqcount_retry(&tk_core.seq, seq));
656
657 ts->tv_nsec = 0;
658 timespec64_add_ns(ts, nsecs);
659
660
661
662
663
664 if (unlikely(timekeeping_suspended))
665 return -EAGAIN;
666 return 0;
667}
668EXPORT_SYMBOL(__getnstimeofday64);
669
670
671
672
673
674
675
676void getnstimeofday64(struct timespec64 *ts)
677{
678 WARN_ON(__getnstimeofday64(ts));
679}
680EXPORT_SYMBOL(getnstimeofday64);
681
682ktime_t ktime_get(void)
683{
684 struct timekeeper *tk = &tk_core.timekeeper;
685 unsigned int seq;
686 ktime_t base;
687 s64 nsecs;
688
689 WARN_ON(timekeeping_suspended);
690
691 do {
692 seq = read_seqcount_begin(&tk_core.seq);
693 base = tk->tkr_mono.base;
694 nsecs = timekeeping_get_ns(&tk->tkr_mono);
695
696 } while (read_seqcount_retry(&tk_core.seq, seq));
697
698 return ktime_add_ns(base, nsecs);
699}
700EXPORT_SYMBOL_GPL(ktime_get);
701
702u32 ktime_get_resolution_ns(void)
703{
704 struct timekeeper *tk = &tk_core.timekeeper;
705 unsigned int seq;
706 u32 nsecs;
707
708 WARN_ON(timekeeping_suspended);
709
710 do {
711 seq = read_seqcount_begin(&tk_core.seq);
712 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
713 } while (read_seqcount_retry(&tk_core.seq, seq));
714
715 return nsecs;
716}
717EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
718
719static ktime_t *offsets[TK_OFFS_MAX] = {
720 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
721 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
722 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
723};
724
725ktime_t ktime_get_with_offset(enum tk_offsets offs)
726{
727 struct timekeeper *tk = &tk_core.timekeeper;
728 unsigned int seq;
729 ktime_t base, *offset = offsets[offs];
730 s64 nsecs;
731
732 WARN_ON(timekeeping_suspended);
733
734 do {
735 seq = read_seqcount_begin(&tk_core.seq);
736 base = ktime_add(tk->tkr_mono.base, *offset);
737 nsecs = timekeeping_get_ns(&tk->tkr_mono);
738
739 } while (read_seqcount_retry(&tk_core.seq, seq));
740
741 return ktime_add_ns(base, nsecs);
742
743}
744EXPORT_SYMBOL_GPL(ktime_get_with_offset);
745
746
747
748
749
750
751ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
752{
753 ktime_t *offset = offsets[offs];
754 unsigned long seq;
755 ktime_t tconv;
756
757 do {
758 seq = read_seqcount_begin(&tk_core.seq);
759 tconv = ktime_add(tmono, *offset);
760 } while (read_seqcount_retry(&tk_core.seq, seq));
761
762 return tconv;
763}
764EXPORT_SYMBOL_GPL(ktime_mono_to_any);
765
766
767
768
769ktime_t ktime_get_raw(void)
770{
771 struct timekeeper *tk = &tk_core.timekeeper;
772 unsigned int seq;
773 ktime_t base;
774 s64 nsecs;
775
776 do {
777 seq = read_seqcount_begin(&tk_core.seq);
778 base = tk->tkr_raw.base;
779 nsecs = timekeeping_get_ns(&tk->tkr_raw);
780
781 } while (read_seqcount_retry(&tk_core.seq, seq));
782
783 return ktime_add_ns(base, nsecs);
784}
785EXPORT_SYMBOL_GPL(ktime_get_raw);
786
787
788
789
790
791
792
793
794
795void ktime_get_ts64(struct timespec64 *ts)
796{
797 struct timekeeper *tk = &tk_core.timekeeper;
798 struct timespec64 tomono;
799 s64 nsec;
800 unsigned int seq;
801
802 WARN_ON(timekeeping_suspended);
803
804 do {
805 seq = read_seqcount_begin(&tk_core.seq);
806 ts->tv_sec = tk->xtime_sec;
807 nsec = timekeeping_get_ns(&tk->tkr_mono);
808 tomono = tk->wall_to_monotonic;
809
810 } while (read_seqcount_retry(&tk_core.seq, seq));
811
812 ts->tv_sec += tomono.tv_sec;
813 ts->tv_nsec = 0;
814 timespec64_add_ns(ts, nsec + tomono.tv_nsec);
815}
816EXPORT_SYMBOL_GPL(ktime_get_ts64);
817
818
819
820
821
822
823
824
825
826
827time64_t ktime_get_seconds(void)
828{
829 struct timekeeper *tk = &tk_core.timekeeper;
830
831 WARN_ON(timekeeping_suspended);
832 return tk->ktime_sec;
833}
834EXPORT_SYMBOL_GPL(ktime_get_seconds);
835
836
837
838
839
840
841
842
843
844
845
846
847time64_t ktime_get_real_seconds(void)
848{
849 struct timekeeper *tk = &tk_core.timekeeper;
850 time64_t seconds;
851 unsigned int seq;
852
853 if (IS_ENABLED(CONFIG_64BIT))
854 return tk->xtime_sec;
855
856 do {
857 seq = read_seqcount_begin(&tk_core.seq);
858 seconds = tk->xtime_sec;
859
860 } while (read_seqcount_retry(&tk_core.seq, seq));
861
862 return seconds;
863}
864EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
865
866
867
868
869
870
871time64_t __ktime_get_real_seconds(void)
872{
873 struct timekeeper *tk = &tk_core.timekeeper;
874
875 return tk->xtime_sec;
876}
877
878
879
880
881
882void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
883{
884 struct timekeeper *tk = &tk_core.timekeeper;
885 unsigned long seq;
886 ktime_t base_raw;
887 ktime_t base_real;
888 s64 nsec_raw;
889 s64 nsec_real;
890 cycle_t now;
891
892 WARN_ON_ONCE(timekeeping_suspended);
893
894 do {
895 seq = read_seqcount_begin(&tk_core.seq);
896
897 now = tk->tkr_mono.read(tk->tkr_mono.clock);
898 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
899 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
900 base_real = ktime_add(tk->tkr_mono.base,
901 tk_core.timekeeper.offs_real);
902 base_raw = tk->tkr_raw.base;
903 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
904 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
905 } while (read_seqcount_retry(&tk_core.seq, seq));
906
907 systime_snapshot->cycles = now;
908 systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
909 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
910}
911EXPORT_SYMBOL_GPL(ktime_get_snapshot);
912
913
914static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
915{
916 u64 tmp, rem;
917
918 tmp = div64_u64_rem(*base, div, &rem);
919
920 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
921 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
922 return -EOVERFLOW;
923 tmp *= mult;
924 rem *= mult;
925
926 do_div(rem, div);
927 *base = tmp + rem;
928 return 0;
929}
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948static int adjust_historical_crosststamp(struct system_time_snapshot *history,
949 cycle_t partial_history_cycles,
950 cycle_t total_history_cycles,
951 bool discontinuity,
952 struct system_device_crosststamp *ts)
953{
954 struct timekeeper *tk = &tk_core.timekeeper;
955 u64 corr_raw, corr_real;
956 bool interp_forward;
957 int ret;
958
959 if (total_history_cycles == 0 || partial_history_cycles == 0)
960 return 0;
961
962
963 interp_forward = partial_history_cycles > total_history_cycles/2 ?
964 true : false;
965 partial_history_cycles = interp_forward ?
966 total_history_cycles - partial_history_cycles :
967 partial_history_cycles;
968
969
970
971
972
973 corr_raw = (u64)ktime_to_ns(
974 ktime_sub(ts->sys_monoraw, history->raw));
975 ret = scale64_check_overflow(partial_history_cycles,
976 total_history_cycles, &corr_raw);
977 if (ret)
978 return ret;
979
980
981
982
983
984
985
986
987 if (discontinuity) {
988 corr_real = mul_u64_u32_div
989 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
990 } else {
991 corr_real = (u64)ktime_to_ns(
992 ktime_sub(ts->sys_realtime, history->real));
993 ret = scale64_check_overflow(partial_history_cycles,
994 total_history_cycles, &corr_real);
995 if (ret)
996 return ret;
997 }
998
999
1000 if (interp_forward) {
1001 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1002 ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1003 } else {
1004 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1005 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1006 }
1007
1008 return 0;
1009}
1010
1011
1012
1013
1014static bool cycle_between(cycle_t before, cycle_t test, cycle_t after)
1015{
1016 if (test > before && test < after)
1017 return true;
1018 if (test < before && before > after)
1019 return true;
1020 return false;
1021}
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034int get_device_system_crosststamp(int (*get_time_fn)
1035 (ktime_t *device_time,
1036 struct system_counterval_t *sys_counterval,
1037 void *ctx),
1038 void *ctx,
1039 struct system_time_snapshot *history_begin,
1040 struct system_device_crosststamp *xtstamp)
1041{
1042 struct system_counterval_t system_counterval;
1043 struct timekeeper *tk = &tk_core.timekeeper;
1044 cycle_t cycles, now, interval_start;
1045 unsigned int clock_was_set_seq = 0;
1046 ktime_t base_real, base_raw;
1047 s64 nsec_real, nsec_raw;
1048 u8 cs_was_changed_seq;
1049 unsigned long seq;
1050 bool do_interp;
1051 int ret;
1052
1053 do {
1054 seq = read_seqcount_begin(&tk_core.seq);
1055
1056
1057
1058
1059 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1060 if (ret)
1061 return ret;
1062
1063
1064
1065
1066
1067
1068 if (tk->tkr_mono.clock != system_counterval.cs)
1069 return -ENODEV;
1070 cycles = system_counterval.cycles;
1071
1072
1073
1074
1075
1076 now = tk->tkr_mono.read(tk->tkr_mono.clock);
1077 interval_start = tk->tkr_mono.cycle_last;
1078 if (!cycle_between(interval_start, cycles, now)) {
1079 clock_was_set_seq = tk->clock_was_set_seq;
1080 cs_was_changed_seq = tk->cs_was_changed_seq;
1081 cycles = interval_start;
1082 do_interp = true;
1083 } else {
1084 do_interp = false;
1085 }
1086
1087 base_real = ktime_add(tk->tkr_mono.base,
1088 tk_core.timekeeper.offs_real);
1089 base_raw = tk->tkr_raw.base;
1090
1091 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1092 system_counterval.cycles);
1093 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1094 system_counterval.cycles);
1095 } while (read_seqcount_retry(&tk_core.seq, seq));
1096
1097 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1098 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1099
1100
1101
1102
1103
1104 if (do_interp) {
1105 cycle_t partial_history_cycles, total_history_cycles;
1106 bool discontinuity;
1107
1108
1109
1110
1111
1112
1113 if (!history_begin ||
1114 !cycle_between(history_begin->cycles,
1115 system_counterval.cycles, cycles) ||
1116 history_begin->cs_was_changed_seq != cs_was_changed_seq)
1117 return -EINVAL;
1118 partial_history_cycles = cycles - system_counterval.cycles;
1119 total_history_cycles = cycles - history_begin->cycles;
1120 discontinuity =
1121 history_begin->clock_was_set_seq != clock_was_set_seq;
1122
1123 ret = adjust_historical_crosststamp(history_begin,
1124 partial_history_cycles,
1125 total_history_cycles,
1126 discontinuity, xtstamp);
1127 if (ret)
1128 return ret;
1129 }
1130
1131 return 0;
1132}
1133EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1134
1135
1136
1137
1138
1139
1140
1141void do_gettimeofday(struct timeval *tv)
1142{
1143 struct timespec64 now;
1144
1145 getnstimeofday64(&now);
1146 tv->tv_sec = now.tv_sec;
1147 tv->tv_usec = now.tv_nsec/1000;
1148}
1149EXPORT_SYMBOL(do_gettimeofday);
1150
1151
1152
1153
1154
1155
1156
1157int do_settimeofday64(const struct timespec64 *ts)
1158{
1159 struct timekeeper *tk = &tk_core.timekeeper;
1160 struct timespec64 ts_delta, xt;
1161 unsigned long flags;
1162 int ret = 0;
1163
1164 if (!timespec64_valid_strict(ts))
1165 return -EINVAL;
1166
1167 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1168 write_seqcount_begin(&tk_core.seq);
1169
1170 timekeeping_forward_now(tk);
1171
1172 xt = tk_xtime(tk);
1173 ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
1174 ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
1175
1176 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
1177 ret = -EINVAL;
1178 goto out;
1179 }
1180
1181 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
1182
1183 tk_set_xtime(tk, ts);
1184out:
1185 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1186
1187 write_seqcount_end(&tk_core.seq);
1188 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1189
1190
1191 clock_was_set();
1192
1193 return ret;
1194}
1195EXPORT_SYMBOL(do_settimeofday64);
1196
1197
1198
1199
1200
1201
1202
1203int timekeeping_inject_offset(struct timespec *ts)
1204{
1205 struct timekeeper *tk = &tk_core.timekeeper;
1206 unsigned long flags;
1207 struct timespec64 ts64, tmp;
1208 int ret = 0;
1209
1210 if (!timespec_inject_offset_valid(ts))
1211 return -EINVAL;
1212
1213 ts64 = timespec_to_timespec64(*ts);
1214
1215 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1216 write_seqcount_begin(&tk_core.seq);
1217
1218 timekeeping_forward_now(tk);
1219
1220
1221 tmp = timespec64_add(tk_xtime(tk), ts64);
1222 if (timespec64_compare(&tk->wall_to_monotonic, &ts64) > 0 ||
1223 !timespec64_valid_strict(&tmp)) {
1224 ret = -EINVAL;
1225 goto error;
1226 }
1227
1228 tk_xtime_add(tk, &ts64);
1229 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64));
1230
1231error:
1232 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1233
1234 write_seqcount_end(&tk_core.seq);
1235 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1236
1237
1238 clock_was_set();
1239
1240 return ret;
1241}
1242EXPORT_SYMBOL(timekeeping_inject_offset);
1243
1244
1245
1246
1247
1248
1249s32 timekeeping_get_tai_offset(void)
1250{
1251 struct timekeeper *tk = &tk_core.timekeeper;
1252 unsigned int seq;
1253 s32 ret;
1254
1255 do {
1256 seq = read_seqcount_begin(&tk_core.seq);
1257 ret = tk->tai_offset;
1258 } while (read_seqcount_retry(&tk_core.seq, seq));
1259
1260 return ret;
1261}
1262
1263
1264
1265
1266
1267static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1268{
1269 tk->tai_offset = tai_offset;
1270 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1271}
1272
1273
1274
1275
1276
1277void timekeeping_set_tai_offset(s32 tai_offset)
1278{
1279 struct timekeeper *tk = &tk_core.timekeeper;
1280 unsigned long flags;
1281
1282 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1283 write_seqcount_begin(&tk_core.seq);
1284 __timekeeping_set_tai_offset(tk, tai_offset);
1285 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1286 write_seqcount_end(&tk_core.seq);
1287 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1288 clock_was_set();
1289}
1290
1291
1292
1293
1294
1295
1296static int change_clocksource(void *data)
1297{
1298 struct timekeeper *tk = &tk_core.timekeeper;
1299 struct clocksource *new, *old;
1300 unsigned long flags;
1301
1302 new = (struct clocksource *) data;
1303
1304 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1305 write_seqcount_begin(&tk_core.seq);
1306
1307 timekeeping_forward_now(tk);
1308
1309
1310
1311
1312 if (try_module_get(new->owner)) {
1313 if (!new->enable || new->enable(new) == 0) {
1314 old = tk->tkr_mono.clock;
1315 tk_setup_internals(tk, new);
1316 if (old->disable)
1317 old->disable(old);
1318 module_put(old->owner);
1319 } else {
1320 module_put(new->owner);
1321 }
1322 }
1323 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1324
1325 write_seqcount_end(&tk_core.seq);
1326 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1327
1328 return 0;
1329}
1330
1331
1332
1333
1334
1335
1336
1337
1338int timekeeping_notify(struct clocksource *clock)
1339{
1340 struct timekeeper *tk = &tk_core.timekeeper;
1341
1342 if (tk->tkr_mono.clock == clock)
1343 return 0;
1344 stop_machine(change_clocksource, clock, NULL);
1345 tick_clock_notify();
1346 return tk->tkr_mono.clock == clock ? 0 : -1;
1347}
1348
1349
1350
1351
1352
1353
1354
1355void getrawmonotonic64(struct timespec64 *ts)
1356{
1357 struct timekeeper *tk = &tk_core.timekeeper;
1358 struct timespec64 ts64;
1359 unsigned long seq;
1360 s64 nsecs;
1361
1362 do {
1363 seq = read_seqcount_begin(&tk_core.seq);
1364 nsecs = timekeeping_get_ns(&tk->tkr_raw);
1365 ts64 = tk->raw_time;
1366
1367 } while (read_seqcount_retry(&tk_core.seq, seq));
1368
1369 timespec64_add_ns(&ts64, nsecs);
1370 *ts = ts64;
1371}
1372EXPORT_SYMBOL(getrawmonotonic64);
1373
1374
1375
1376
1377
1378int timekeeping_valid_for_hres(void)
1379{
1380 struct timekeeper *tk = &tk_core.timekeeper;
1381 unsigned long seq;
1382 int ret;
1383
1384 do {
1385 seq = read_seqcount_begin(&tk_core.seq);
1386
1387 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1388
1389 } while (read_seqcount_retry(&tk_core.seq, seq));
1390
1391 return ret;
1392}
1393
1394
1395
1396
1397u64 timekeeping_max_deferment(void)
1398{
1399 struct timekeeper *tk = &tk_core.timekeeper;
1400 unsigned long seq;
1401 u64 ret;
1402
1403 do {
1404 seq = read_seqcount_begin(&tk_core.seq);
1405
1406 ret = tk->tkr_mono.clock->max_idle_ns;
1407
1408 } while (read_seqcount_retry(&tk_core.seq, seq));
1409
1410 return ret;
1411}
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422void __weak read_persistent_clock(struct timespec *ts)
1423{
1424 ts->tv_sec = 0;
1425 ts->tv_nsec = 0;
1426}
1427
1428void __weak read_persistent_clock64(struct timespec64 *ts64)
1429{
1430 struct timespec ts;
1431
1432 read_persistent_clock(&ts);
1433 *ts64 = timespec_to_timespec64(ts);
1434}
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445void __weak read_boot_clock64(struct timespec64 *ts)
1446{
1447 ts->tv_sec = 0;
1448 ts->tv_nsec = 0;
1449}
1450
1451
1452static bool sleeptime_injected;
1453
1454
1455static bool persistent_clock_exists;
1456
1457
1458
1459
1460void __init timekeeping_init(void)
1461{
1462 struct timekeeper *tk = &tk_core.timekeeper;
1463 struct clocksource *clock;
1464 unsigned long flags;
1465 struct timespec64 now, boot, tmp;
1466
1467 read_persistent_clock64(&now);
1468 if (!timespec64_valid_strict(&now)) {
1469 pr_warn("WARNING: Persistent clock returned invalid value!\n"
1470 " Check your CMOS/BIOS settings.\n");
1471 now.tv_sec = 0;
1472 now.tv_nsec = 0;
1473 } else if (now.tv_sec || now.tv_nsec)
1474 persistent_clock_exists = true;
1475
1476 read_boot_clock64(&boot);
1477 if (!timespec64_valid_strict(&boot)) {
1478 pr_warn("WARNING: Boot clock returned invalid value!\n"
1479 " Check your CMOS/BIOS settings.\n");
1480 boot.tv_sec = 0;
1481 boot.tv_nsec = 0;
1482 }
1483
1484 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1485 write_seqcount_begin(&tk_core.seq);
1486 ntp_init();
1487
1488 clock = clocksource_default_clock();
1489 if (clock->enable)
1490 clock->enable(clock);
1491 tk_setup_internals(tk, clock);
1492
1493 tk_set_xtime(tk, &now);
1494 tk->raw_time.tv_sec = 0;
1495 tk->raw_time.tv_nsec = 0;
1496 if (boot.tv_sec == 0 && boot.tv_nsec == 0)
1497 boot = tk_xtime(tk);
1498
1499 set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec);
1500 tk_set_wall_to_mono(tk, tmp);
1501
1502 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1503
1504 write_seqcount_end(&tk_core.seq);
1505 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1506}
1507
1508
1509static struct timespec64 timekeeping_suspend_time;
1510
1511
1512
1513
1514
1515
1516
1517
1518static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1519 struct timespec64 *delta)
1520{
1521 if (!timespec64_valid_strict(delta)) {
1522 printk_deferred(KERN_WARNING
1523 "__timekeeping_inject_sleeptime: Invalid "
1524 "sleep delta value!\n");
1525 return;
1526 }
1527 tk_xtime_add(tk, delta);
1528 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1529 tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1530 tk_debug_account_sleep_time(delta);
1531}
1532
1533#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550bool timekeeping_rtc_skipresume(void)
1551{
1552 return sleeptime_injected;
1553}
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564bool timekeeping_rtc_skipsuspend(void)
1565{
1566 return persistent_clock_exists;
1567}
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580void timekeeping_inject_sleeptime64(struct timespec64 *delta)
1581{
1582 struct timekeeper *tk = &tk_core.timekeeper;
1583 unsigned long flags;
1584
1585 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1586 write_seqcount_begin(&tk_core.seq);
1587
1588 timekeeping_forward_now(tk);
1589
1590 __timekeeping_inject_sleeptime(tk, delta);
1591
1592 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1593
1594 write_seqcount_end(&tk_core.seq);
1595 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1596
1597
1598 clock_was_set();
1599}
1600#endif
1601
1602
1603
1604
1605void timekeeping_resume(void)
1606{
1607 struct timekeeper *tk = &tk_core.timekeeper;
1608 struct clocksource *clock = tk->tkr_mono.clock;
1609 unsigned long flags;
1610 struct timespec64 ts_new, ts_delta;
1611 cycle_t cycle_now, cycle_delta;
1612
1613 sleeptime_injected = false;
1614 read_persistent_clock64(&ts_new);
1615
1616 clockevents_resume();
1617 clocksource_resume();
1618
1619 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1620 write_seqcount_begin(&tk_core.seq);
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634 cycle_now = tk->tkr_mono.read(clock);
1635 if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
1636 cycle_now > tk->tkr_mono.cycle_last) {
1637 u64 num, max = ULLONG_MAX;
1638 u32 mult = clock->mult;
1639 u32 shift = clock->shift;
1640 s64 nsec = 0;
1641
1642 cycle_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last,
1643 tk->tkr_mono.mask);
1644
1645
1646
1647
1648
1649
1650 do_div(max, mult);
1651 if (cycle_delta > max) {
1652 num = div64_u64(cycle_delta, max);
1653 nsec = (((u64) max * mult) >> shift) * num;
1654 cycle_delta -= num * max;
1655 }
1656 nsec += ((u64) cycle_delta * mult) >> shift;
1657
1658 ts_delta = ns_to_timespec64(nsec);
1659 sleeptime_injected = true;
1660 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1661 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1662 sleeptime_injected = true;
1663 }
1664
1665 if (sleeptime_injected)
1666 __timekeeping_inject_sleeptime(tk, &ts_delta);
1667
1668
1669 tk->tkr_mono.cycle_last = cycle_now;
1670 tk->tkr_raw.cycle_last = cycle_now;
1671
1672 tk->ntp_error = 0;
1673 timekeeping_suspended = 0;
1674 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1675 write_seqcount_end(&tk_core.seq);
1676 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1677
1678 touch_softlockup_watchdog();
1679
1680 tick_resume();
1681 hrtimers_resume();
1682}
1683
1684int timekeeping_suspend(void)
1685{
1686 struct timekeeper *tk = &tk_core.timekeeper;
1687 unsigned long flags;
1688 struct timespec64 delta, delta_delta;
1689 static struct timespec64 old_delta;
1690
1691 read_persistent_clock64(&timekeeping_suspend_time);
1692
1693
1694
1695
1696
1697
1698 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1699 persistent_clock_exists = true;
1700
1701 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1702 write_seqcount_begin(&tk_core.seq);
1703 timekeeping_forward_now(tk);
1704 timekeeping_suspended = 1;
1705
1706 if (persistent_clock_exists) {
1707
1708
1709
1710
1711
1712
1713 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1714 delta_delta = timespec64_sub(delta, old_delta);
1715 if (abs(delta_delta.tv_sec) >= 2) {
1716
1717
1718
1719
1720 old_delta = delta;
1721 } else {
1722
1723 timekeeping_suspend_time =
1724 timespec64_add(timekeeping_suspend_time, delta_delta);
1725 }
1726 }
1727
1728 timekeeping_update(tk, TK_MIRROR);
1729 halt_fast_timekeeper(tk);
1730 write_seqcount_end(&tk_core.seq);
1731 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1732
1733 tick_suspend();
1734 clocksource_suspend();
1735 clockevents_suspend();
1736
1737 return 0;
1738}
1739
1740
1741static struct syscore_ops timekeeping_syscore_ops = {
1742 .resume = timekeeping_resume,
1743 .suspend = timekeeping_suspend,
1744};
1745
1746static int __init timekeeping_init_ops(void)
1747{
1748 register_syscore_ops(&timekeeping_syscore_ops);
1749 return 0;
1750}
1751device_initcall(timekeeping_init_ops);
1752
1753
1754
1755
1756static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1757 s64 offset,
1758 bool negative,
1759 int adj_scale)
1760{
1761 s64 interval = tk->cycle_interval;
1762 s32 mult_adj = 1;
1763
1764 if (negative) {
1765 mult_adj = -mult_adj;
1766 interval = -interval;
1767 offset = -offset;
1768 }
1769 mult_adj <<= adj_scale;
1770 interval <<= adj_scale;
1771 offset <<= adj_scale;
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1823
1824 WARN_ON_ONCE(1);
1825 return;
1826 }
1827
1828 tk->tkr_mono.mult += mult_adj;
1829 tk->xtime_interval += interval;
1830 tk->tkr_mono.xtime_nsec -= offset;
1831 tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
1832}
1833
1834
1835
1836
1837
1838static __always_inline void timekeeping_freqadjust(struct timekeeper *tk,
1839 s64 offset)
1840{
1841 s64 interval = tk->cycle_interval;
1842 s64 xinterval = tk->xtime_interval;
1843 u32 base = tk->tkr_mono.clock->mult;
1844 u32 max = tk->tkr_mono.clock->maxadj;
1845 u32 cur_adj = tk->tkr_mono.mult;
1846 s64 tick_error;
1847 bool negative;
1848 u32 adj_scale;
1849
1850
1851 if (tk->ntp_err_mult)
1852 xinterval -= tk->cycle_interval;
1853
1854 tk->ntp_tick = ntp_tick_length();
1855
1856
1857 tick_error = ntp_tick_length() >> tk->ntp_error_shift;
1858 tick_error -= (xinterval + tk->xtime_remainder);
1859
1860
1861 if (likely((tick_error >= 0) && (tick_error <= interval)))
1862 return;
1863
1864
1865 negative = (tick_error < 0);
1866
1867
1868 if (negative && (cur_adj - 1) <= (base - max))
1869 return;
1870 if (!negative && (cur_adj + 1) >= (base + max))
1871 return;
1872
1873
1874
1875
1876
1877 adj_scale = 0;
1878 tick_error = abs(tick_error);
1879 while (tick_error > interval) {
1880 u32 adj = 1 << (adj_scale + 1);
1881
1882
1883 if (negative && (cur_adj - adj) <= (base - max))
1884 break;
1885 if (!negative && (cur_adj + adj) >= (base + max))
1886 break;
1887
1888 adj_scale++;
1889 tick_error >>= 1;
1890 }
1891
1892
1893 timekeeping_apply_adjustment(tk, offset, negative, adj_scale);
1894}
1895
1896
1897
1898
1899
1900static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1901{
1902
1903 timekeeping_freqadjust(tk, offset);
1904
1905
1906 if (!tk->ntp_err_mult && (tk->ntp_error > 0)) {
1907 tk->ntp_err_mult = 1;
1908 timekeeping_apply_adjustment(tk, offset, 0, 0);
1909 } else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) {
1910
1911 timekeeping_apply_adjustment(tk, offset, 1, 0);
1912 tk->ntp_err_mult = 0;
1913 }
1914
1915 if (unlikely(tk->tkr_mono.clock->maxadj &&
1916 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
1917 > tk->tkr_mono.clock->maxadj))) {
1918 printk_once(KERN_WARNING
1919 "Adjusting %s more than 11%% (%ld vs %ld)\n",
1920 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
1921 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
1922 }
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
1939 s64 neg = -(s64)tk->tkr_mono.xtime_nsec;
1940 tk->tkr_mono.xtime_nsec = 0;
1941 tk->ntp_error += neg << tk->ntp_error_shift;
1942 }
1943}
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1954{
1955 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
1956 unsigned int clock_set = 0;
1957
1958 while (tk->tkr_mono.xtime_nsec >= nsecps) {
1959 int leap;
1960
1961 tk->tkr_mono.xtime_nsec -= nsecps;
1962 tk->xtime_sec++;
1963
1964
1965 leap = second_overflow(tk->xtime_sec);
1966 if (unlikely(leap)) {
1967 struct timespec64 ts;
1968
1969 tk->xtime_sec += leap;
1970
1971 ts.tv_sec = leap;
1972 ts.tv_nsec = 0;
1973 tk_set_wall_to_mono(tk,
1974 timespec64_sub(tk->wall_to_monotonic, ts));
1975
1976 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1977
1978 clock_set = TK_CLOCK_WAS_SET;
1979 }
1980 }
1981 return clock_set;
1982}
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1994 u32 shift,
1995 unsigned int *clock_set)
1996{
1997 cycle_t interval = tk->cycle_interval << shift;
1998 u64 raw_nsecs;
1999
2000
2001 if (offset < interval)
2002 return offset;
2003
2004
2005 offset -= interval;
2006 tk->tkr_mono.cycle_last += interval;
2007 tk->tkr_raw.cycle_last += interval;
2008
2009 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2010 *clock_set |= accumulate_nsecs_to_secs(tk);
2011
2012
2013 raw_nsecs = (u64)tk->raw_interval << shift;
2014 raw_nsecs += tk->raw_time.tv_nsec;
2015 if (raw_nsecs >= NSEC_PER_SEC) {
2016 u64 raw_secs = raw_nsecs;
2017 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
2018 tk->raw_time.tv_sec += raw_secs;
2019 }
2020 tk->raw_time.tv_nsec = raw_nsecs;
2021
2022
2023 tk->ntp_error += tk->ntp_tick << shift;
2024 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2025 (tk->ntp_error_shift + shift);
2026
2027 return offset;
2028}
2029
2030
2031
2032
2033
2034void update_wall_time(void)
2035{
2036 struct timekeeper *real_tk = &tk_core.timekeeper;
2037 struct timekeeper *tk = &shadow_timekeeper;
2038 cycle_t offset;
2039 int shift = 0, maxshift;
2040 unsigned int clock_set = 0;
2041 unsigned long flags;
2042
2043 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2044
2045
2046 if (unlikely(timekeeping_suspended))
2047 goto out;
2048
2049#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
2050 offset = real_tk->cycle_interval;
2051#else
2052 offset = clocksource_delta(tk->tkr_mono.read(tk->tkr_mono.clock),
2053 tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
2054#endif
2055
2056
2057 if (offset < real_tk->cycle_interval)
2058 goto out;
2059
2060
2061 timekeeping_check_update(real_tk, offset);
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071 shift = ilog2(offset) - ilog2(tk->cycle_interval);
2072 shift = max(0, shift);
2073
2074 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
2075 shift = min(shift, maxshift);
2076 while (offset >= tk->cycle_interval) {
2077 offset = logarithmic_accumulation(tk, offset, shift,
2078 &clock_set);
2079 if (offset < tk->cycle_interval<<shift)
2080 shift--;
2081 }
2082
2083
2084 timekeeping_adjust(tk, offset);
2085
2086
2087
2088
2089
2090 old_vsyscall_fixup(tk);
2091
2092
2093
2094
2095
2096 clock_set |= accumulate_nsecs_to_secs(tk);
2097
2098 write_seqcount_begin(&tk_core.seq);
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109 timekeeping_update(tk, clock_set);
2110 memcpy(real_tk, tk, sizeof(*tk));
2111
2112 write_seqcount_end(&tk_core.seq);
2113out:
2114 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2115 if (clock_set)
2116
2117 clock_was_set_delayed();
2118}
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131void getboottime64(struct timespec64 *ts)
2132{
2133 struct timekeeper *tk = &tk_core.timekeeper;
2134 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2135
2136 *ts = ktime_to_timespec64(t);
2137}
2138EXPORT_SYMBOL_GPL(getboottime64);
2139
2140unsigned long get_seconds(void)
2141{
2142 struct timekeeper *tk = &tk_core.timekeeper;
2143
2144 return tk->xtime_sec;
2145}
2146EXPORT_SYMBOL(get_seconds);
2147
2148struct timespec __current_kernel_time(void)
2149{
2150 struct timekeeper *tk = &tk_core.timekeeper;
2151
2152 return timespec64_to_timespec(tk_xtime(tk));
2153}
2154
2155struct timespec64 current_kernel_time64(void)
2156{
2157 struct timekeeper *tk = &tk_core.timekeeper;
2158 struct timespec64 now;
2159 unsigned long seq;
2160
2161 do {
2162 seq = read_seqcount_begin(&tk_core.seq);
2163
2164 now = tk_xtime(tk);
2165 } while (read_seqcount_retry(&tk_core.seq, seq));
2166
2167 return now;
2168}
2169EXPORT_SYMBOL(current_kernel_time64);
2170
2171struct timespec64 get_monotonic_coarse64(void)
2172{
2173 struct timekeeper *tk = &tk_core.timekeeper;
2174 struct timespec64 now, mono;
2175 unsigned long seq;
2176
2177 do {
2178 seq = read_seqcount_begin(&tk_core.seq);
2179
2180 now = tk_xtime(tk);
2181 mono = tk->wall_to_monotonic;
2182 } while (read_seqcount_retry(&tk_core.seq, seq));
2183
2184 set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec,
2185 now.tv_nsec + mono.tv_nsec);
2186
2187 return now;
2188}
2189
2190
2191
2192
2193void do_timer(unsigned long ticks)
2194{
2195 jiffies_64 += ticks;
2196 calc_global_load(ticks);
2197}
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2213 ktime_t *offs_boot, ktime_t *offs_tai)
2214{
2215 struct timekeeper *tk = &tk_core.timekeeper;
2216 unsigned int seq;
2217 ktime_t base;
2218 u64 nsecs;
2219
2220 do {
2221 seq = read_seqcount_begin(&tk_core.seq);
2222
2223 base = tk->tkr_mono.base;
2224 nsecs = timekeeping_get_ns(&tk->tkr_mono);
2225 base = ktime_add_ns(base, nsecs);
2226
2227 if (*cwsseq != tk->clock_was_set_seq) {
2228 *cwsseq = tk->clock_was_set_seq;
2229 *offs_real = tk->offs_real;
2230 *offs_boot = tk->offs_boot;
2231 *offs_tai = tk->offs_tai;
2232 }
2233
2234
2235 if (unlikely(base.tv64 >= tk->next_leap_ktime.tv64))
2236 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2237
2238 } while (read_seqcount_retry(&tk_core.seq, seq));
2239
2240 return base;
2241}
2242
2243
2244
2245
2246int do_adjtimex(struct timex *txc)
2247{
2248 struct timekeeper *tk = &tk_core.timekeeper;
2249 unsigned long flags;
2250 struct timespec64 ts;
2251 s32 orig_tai, tai;
2252 int ret;
2253
2254
2255 ret = ntp_validate_timex(txc);
2256 if (ret)
2257 return ret;
2258
2259 if (txc->modes & ADJ_SETOFFSET) {
2260 struct timespec delta;
2261 delta.tv_sec = txc->time.tv_sec;
2262 delta.tv_nsec = txc->time.tv_usec;
2263 if (!(txc->modes & ADJ_NANO))
2264 delta.tv_nsec *= 1000;
2265 ret = timekeeping_inject_offset(&delta);
2266 if (ret)
2267 return ret;
2268 }
2269
2270 getnstimeofday64(&ts);
2271
2272 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2273 write_seqcount_begin(&tk_core.seq);
2274
2275 orig_tai = tai = tk->tai_offset;
2276 ret = __do_adjtimex(txc, &ts, &tai);
2277
2278 if (tai != orig_tai) {
2279 __timekeeping_set_tai_offset(tk, tai);
2280 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2281 }
2282 tk_update_leap_state(tk);
2283
2284 write_seqcount_end(&tk_core.seq);
2285 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2286
2287 if (tai != orig_tai)
2288 clock_was_set();
2289
2290 ntp_notify_cmos_timer();
2291
2292 return ret;
2293}
2294
2295#ifdef CONFIG_NTP_PPS
2296
2297
2298
2299void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2300{
2301 unsigned long flags;
2302
2303 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2304 write_seqcount_begin(&tk_core.seq);
2305
2306 __hardpps(phase_ts, raw_ts);
2307
2308 write_seqcount_end(&tk_core.seq);
2309 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2310}
2311EXPORT_SYMBOL(hardpps);
2312#endif
2313
2314
2315
2316
2317
2318
2319
2320void xtime_update(unsigned long ticks)
2321{
2322 write_seqlock(&jiffies_lock);
2323 do_timer(ticks);
2324 write_sequnlock(&jiffies_lock);
2325 update_wall_time();
2326}
2327