1
2
3
4
5
6#include <linux/timekeeper_internal.h>
7#include <linux/module.h>
8#include <linux/interrupt.h>
9#include <linux/percpu.h>
10#include <linux/init.h>
11#include <linux/mm.h>
12#include <linux/nmi.h>
13#include <linux/sched.h>
14#include <linux/sched/loadavg.h>
15#include <linux/sched/clock.h>
16#include <linux/syscore_ops.h>
17#include <linux/clocksource.h>
18#include <linux/jiffies.h>
19#include <linux/time.h>
20#include <linux/tick.h>
21#include <linux/stop_machine.h>
22#include <linux/pvclock_gtod.h>
23#include <linux/compiler.h>
24#include <linux/audit.h>
25
26#include "tick-internal.h"
27#include "ntp_internal.h"
28#include "timekeeping_internal.h"
29
30#define TK_CLEAR_NTP (1 << 0)
31#define TK_MIRROR (1 << 1)
32#define TK_CLOCK_WAS_SET (1 << 2)
33
34enum timekeeping_adv_mode {
35
36 TK_ADV_TICK,
37
38
39 TK_ADV_FREQ
40};
41
42DEFINE_RAW_SPINLOCK(timekeeper_lock);
43
44
45
46
47
48static struct {
49 seqcount_raw_spinlock_t seq;
50 struct timekeeper timekeeper;
51} tk_core ____cacheline_aligned = {
52 .seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_core.seq, &timekeeper_lock),
53};
54
55static struct timekeeper shadow_timekeeper;
56
57
58int __read_mostly timekeeping_suspended;
59
60
61
62
63
64
65
66
67
68
69struct tk_fast {
70 seqcount_latch_t seq;
71 struct tk_read_base base[2];
72};
73
74
75static u64 cycles_at_suspend;
76
77static u64 dummy_clock_read(struct clocksource *cs)
78{
79 if (timekeeping_suspended)
80 return cycles_at_suspend;
81 return local_clock();
82}
83
84static struct clocksource dummy_clock = {
85 .read = dummy_clock_read,
86};
87
88
89
90
91
92
93
94
95#define FAST_TK_INIT \
96 { \
97 .clock = &dummy_clock, \
98 .mask = CLOCKSOURCE_MASK(64), \
99 .mult = 1, \
100 .shift = 0, \
101 }
102
103static struct tk_fast tk_fast_mono ____cacheline_aligned = {
104 .seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
105 .base[0] = FAST_TK_INIT,
106 .base[1] = FAST_TK_INIT,
107};
108
109static struct tk_fast tk_fast_raw ____cacheline_aligned = {
110 .seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
111 .base[0] = FAST_TK_INIT,
112 .base[1] = FAST_TK_INIT,
113};
114
115static inline void tk_normalize_xtime(struct timekeeper *tk)
116{
117 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
118 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
119 tk->xtime_sec++;
120 }
121 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
122 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
123 tk->raw_sec++;
124 }
125}
126
127static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
128{
129 struct timespec64 ts;
130
131 ts.tv_sec = tk->xtime_sec;
132 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
133 return ts;
134}
135
136static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
137{
138 tk->xtime_sec = ts->tv_sec;
139 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
140}
141
142static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
143{
144 tk->xtime_sec += ts->tv_sec;
145 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
146 tk_normalize_xtime(tk);
147}
148
149static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
150{
151 struct timespec64 tmp;
152
153
154
155
156
157 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
158 -tk->wall_to_monotonic.tv_nsec);
159 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
160 tk->wall_to_monotonic = wtm;
161 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
162 tk->offs_real = timespec64_to_ktime(tmp);
163 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
164}
165
166static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
167{
168 tk->offs_boot = ktime_add(tk->offs_boot, delta);
169
170
171
172
173 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
174}
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189static inline u64 tk_clock_read(const struct tk_read_base *tkr)
190{
191 struct clocksource *clock = READ_ONCE(tkr->clock);
192
193 return clock->read(clock);
194}
195
196#ifdef CONFIG_DEBUG_TIMEKEEPING
197#define WARNING_FREQ (HZ*300)
198
199static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
200{
201
202 u64 max_cycles = tk->tkr_mono.clock->max_cycles;
203 const char *name = tk->tkr_mono.clock->name;
204
205 if (offset > max_cycles) {
206 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
207 offset, name, max_cycles);
208 printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
209 } else {
210 if (offset > (max_cycles >> 1)) {
211 printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
212 offset, name, max_cycles >> 1);
213 printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
214 }
215 }
216
217 if (tk->underflow_seen) {
218 if (jiffies - tk->last_warning > WARNING_FREQ) {
219 printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
220 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
221 printk_deferred(" Your kernel is probably still fine.\n");
222 tk->last_warning = jiffies;
223 }
224 tk->underflow_seen = 0;
225 }
226
227 if (tk->overflow_seen) {
228 if (jiffies - tk->last_warning > WARNING_FREQ) {
229 printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
230 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
231 printk_deferred(" Your kernel is probably still fine.\n");
232 tk->last_warning = jiffies;
233 }
234 tk->overflow_seen = 0;
235 }
236}
237
238static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
239{
240 struct timekeeper *tk = &tk_core.timekeeper;
241 u64 now, last, mask, max, delta;
242 unsigned int seq;
243
244
245
246
247
248
249
250
251 do {
252 seq = read_seqcount_begin(&tk_core.seq);
253 now = tk_clock_read(tkr);
254 last = tkr->cycle_last;
255 mask = tkr->mask;
256 max = tkr->clock->max_cycles;
257 } while (read_seqcount_retry(&tk_core.seq, seq));
258
259 delta = clocksource_delta(now, last, mask);
260
261
262
263
264
265 if (unlikely((~delta & mask) < (mask >> 3))) {
266 tk->underflow_seen = 1;
267 delta = 0;
268 }
269
270
271 if (unlikely(delta > max)) {
272 tk->overflow_seen = 1;
273 delta = tkr->clock->max_cycles;
274 }
275
276 return delta;
277}
278#else
279static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
280{
281}
282static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
283{
284 u64 cycle_now, delta;
285
286
287 cycle_now = tk_clock_read(tkr);
288
289
290 delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
291
292 return delta;
293}
294#endif
295
296
297
298
299
300
301
302
303
304
305
306
307static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
308{
309 u64 interval;
310 u64 tmp, ntpinterval;
311 struct clocksource *old_clock;
312
313 ++tk->cs_was_changed_seq;
314 old_clock = tk->tkr_mono.clock;
315 tk->tkr_mono.clock = clock;
316 tk->tkr_mono.mask = clock->mask;
317 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
318
319 tk->tkr_raw.clock = clock;
320 tk->tkr_raw.mask = clock->mask;
321 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
322
323
324 tmp = NTP_INTERVAL_LENGTH;
325 tmp <<= clock->shift;
326 ntpinterval = tmp;
327 tmp += clock->mult/2;
328 do_div(tmp, clock->mult);
329 if (tmp == 0)
330 tmp = 1;
331
332 interval = (u64) tmp;
333 tk->cycle_interval = interval;
334
335
336 tk->xtime_interval = interval * clock->mult;
337 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
338 tk->raw_interval = interval * clock->mult;
339
340
341 if (old_clock) {
342 int shift_change = clock->shift - old_clock->shift;
343 if (shift_change < 0) {
344 tk->tkr_mono.xtime_nsec >>= -shift_change;
345 tk->tkr_raw.xtime_nsec >>= -shift_change;
346 } else {
347 tk->tkr_mono.xtime_nsec <<= shift_change;
348 tk->tkr_raw.xtime_nsec <<= shift_change;
349 }
350 }
351
352 tk->tkr_mono.shift = clock->shift;
353 tk->tkr_raw.shift = clock->shift;
354
355 tk->ntp_error = 0;
356 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
357 tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
358
359
360
361
362
363
364 tk->tkr_mono.mult = clock->mult;
365 tk->tkr_raw.mult = clock->mult;
366 tk->ntp_err_mult = 0;
367 tk->skip_second_overflow = 0;
368}
369
370
371
372static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
373{
374 u64 nsec;
375
376 nsec = delta * tkr->mult + tkr->xtime_nsec;
377 nsec >>= tkr->shift;
378
379 return nsec;
380}
381
382static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
383{
384 u64 delta;
385
386 delta = timekeeping_get_delta(tkr);
387 return timekeeping_delta_to_ns(tkr, delta);
388}
389
390static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
391{
392 u64 delta;
393
394
395 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
396 return timekeeping_delta_to_ns(tkr, delta);
397}
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414static void update_fast_timekeeper(const struct tk_read_base *tkr,
415 struct tk_fast *tkf)
416{
417 struct tk_read_base *base = tkf->base;
418
419
420 raw_write_seqcount_latch(&tkf->seq);
421
422
423 memcpy(base, tkr, sizeof(*base));
424
425
426 raw_write_seqcount_latch(&tkf->seq);
427
428
429 memcpy(base + 1, base, sizeof(*base));
430}
431
432static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
433{
434 struct tk_read_base *tkr;
435 unsigned int seq;
436 u64 now;
437
438 do {
439 seq = raw_read_seqcount_latch(&tkf->seq);
440 tkr = tkf->base + (seq & 0x01);
441 now = ktime_to_ns(tkr->base);
442
443 now += timekeeping_delta_to_ns(tkr,
444 clocksource_delta(
445 tk_clock_read(tkr),
446 tkr->cycle_last,
447 tkr->mask));
448 } while (read_seqcount_latch_retry(&tkf->seq, seq));
449
450 return now;
451}
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485u64 ktime_get_mono_fast_ns(void)
486{
487 return __ktime_get_fast_ns(&tk_fast_mono);
488}
489EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
490
491
492
493
494
495
496
497u64 ktime_get_raw_fast_ns(void)
498{
499 return __ktime_get_fast_ns(&tk_fast_raw);
500}
501EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527u64 notrace ktime_get_boot_fast_ns(void)
528{
529 struct timekeeper *tk = &tk_core.timekeeper;
530
531 return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
532}
533EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
534
535static __always_inline u64 __ktime_get_real_fast(struct tk_fast *tkf, u64 *mono)
536{
537 struct tk_read_base *tkr;
538 u64 basem, baser, delta;
539 unsigned int seq;
540
541 do {
542 seq = raw_read_seqcount_latch(&tkf->seq);
543 tkr = tkf->base + (seq & 0x01);
544 basem = ktime_to_ns(tkr->base);
545 baser = ktime_to_ns(tkr->base_real);
546
547 delta = timekeeping_delta_to_ns(tkr,
548 clocksource_delta(tk_clock_read(tkr),
549 tkr->cycle_last, tkr->mask));
550 } while (read_seqcount_latch_retry(&tkf->seq, seq));
551
552 if (mono)
553 *mono = basem + delta;
554 return baser + delta;
555}
556
557
558
559
560
561
562u64 ktime_get_real_fast_ns(void)
563{
564 return __ktime_get_real_fast(&tk_fast_mono, NULL);
565}
566EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614void ktime_get_fast_timestamps(struct ktime_timestamps *snapshot)
615{
616 struct timekeeper *tk = &tk_core.timekeeper;
617
618 snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono);
619 snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot));
620}
621
622
623
624
625
626
627
628
629
630
631
632static void halt_fast_timekeeper(const struct timekeeper *tk)
633{
634 static struct tk_read_base tkr_dummy;
635 const struct tk_read_base *tkr = &tk->tkr_mono;
636
637 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
638 cycles_at_suspend = tk_clock_read(tkr);
639 tkr_dummy.clock = &dummy_clock;
640 tkr_dummy.base_real = tkr->base + tk->offs_real;
641 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
642
643 tkr = &tk->tkr_raw;
644 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
645 tkr_dummy.clock = &dummy_clock;
646 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
647}
648
649static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
650
651static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
652{
653 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
654}
655
656
657
658
659
660int pvclock_gtod_register_notifier(struct notifier_block *nb)
661{
662 struct timekeeper *tk = &tk_core.timekeeper;
663 unsigned long flags;
664 int ret;
665
666 raw_spin_lock_irqsave(&timekeeper_lock, flags);
667 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
668 update_pvclock_gtod(tk, true);
669 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
670
671 return ret;
672}
673EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
674
675
676
677
678
679
680int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
681{
682 unsigned long flags;
683 int ret;
684
685 raw_spin_lock_irqsave(&timekeeper_lock, flags);
686 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
687 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
688
689 return ret;
690}
691EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
692
693
694
695
696static inline void tk_update_leap_state(struct timekeeper *tk)
697{
698 tk->next_leap_ktime = ntp_get_next_leap();
699 if (tk->next_leap_ktime != KTIME_MAX)
700
701 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
702}
703
704
705
706
707static inline void tk_update_ktime_data(struct timekeeper *tk)
708{
709 u64 seconds;
710 u32 nsec;
711
712
713
714
715
716
717
718
719 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
720 nsec = (u32) tk->wall_to_monotonic.tv_nsec;
721 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
722
723
724
725
726
727
728 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
729 if (nsec >= NSEC_PER_SEC)
730 seconds++;
731 tk->ktime_sec = seconds;
732
733
734 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
735}
736
737
738static void timekeeping_update(struct timekeeper *tk, unsigned int action)
739{
740 if (action & TK_CLEAR_NTP) {
741 tk->ntp_error = 0;
742 ntp_clear();
743 }
744
745 tk_update_leap_state(tk);
746 tk_update_ktime_data(tk);
747
748 update_vsyscall(tk);
749 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
750
751 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
752 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
753 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
754
755 if (action & TK_CLOCK_WAS_SET)
756 tk->clock_was_set_seq++;
757
758
759
760
761
762 if (action & TK_MIRROR)
763 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
764 sizeof(tk_core.timekeeper));
765}
766
767
768
769
770
771
772
773
774
775static void timekeeping_forward_now(struct timekeeper *tk)
776{
777 u64 cycle_now, delta;
778
779 cycle_now = tk_clock_read(&tk->tkr_mono);
780 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
781 tk->tkr_mono.cycle_last = cycle_now;
782 tk->tkr_raw.cycle_last = cycle_now;
783
784 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
785 tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
786
787 tk_normalize_xtime(tk);
788}
789
790
791
792
793
794
795
796void ktime_get_real_ts64(struct timespec64 *ts)
797{
798 struct timekeeper *tk = &tk_core.timekeeper;
799 unsigned int seq;
800 u64 nsecs;
801
802 WARN_ON(timekeeping_suspended);
803
804 do {
805 seq = read_seqcount_begin(&tk_core.seq);
806
807 ts->tv_sec = tk->xtime_sec;
808 nsecs = timekeeping_get_ns(&tk->tkr_mono);
809
810 } while (read_seqcount_retry(&tk_core.seq, seq));
811
812 ts->tv_nsec = 0;
813 timespec64_add_ns(ts, nsecs);
814}
815EXPORT_SYMBOL(ktime_get_real_ts64);
816
817ktime_t ktime_get(void)
818{
819 struct timekeeper *tk = &tk_core.timekeeper;
820 unsigned int seq;
821 ktime_t base;
822 u64 nsecs;
823
824 WARN_ON(timekeeping_suspended);
825
826 do {
827 seq = read_seqcount_begin(&tk_core.seq);
828 base = tk->tkr_mono.base;
829 nsecs = timekeeping_get_ns(&tk->tkr_mono);
830
831 } while (read_seqcount_retry(&tk_core.seq, seq));
832
833 return ktime_add_ns(base, nsecs);
834}
835EXPORT_SYMBOL_GPL(ktime_get);
836
837u32 ktime_get_resolution_ns(void)
838{
839 struct timekeeper *tk = &tk_core.timekeeper;
840 unsigned int seq;
841 u32 nsecs;
842
843 WARN_ON(timekeeping_suspended);
844
845 do {
846 seq = read_seqcount_begin(&tk_core.seq);
847 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
848 } while (read_seqcount_retry(&tk_core.seq, seq));
849
850 return nsecs;
851}
852EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
853
854static ktime_t *offsets[TK_OFFS_MAX] = {
855 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
856 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
857 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
858};
859
860ktime_t ktime_get_with_offset(enum tk_offsets offs)
861{
862 struct timekeeper *tk = &tk_core.timekeeper;
863 unsigned int seq;
864 ktime_t base, *offset = offsets[offs];
865 u64 nsecs;
866
867 WARN_ON(timekeeping_suspended);
868
869 do {
870 seq = read_seqcount_begin(&tk_core.seq);
871 base = ktime_add(tk->tkr_mono.base, *offset);
872 nsecs = timekeeping_get_ns(&tk->tkr_mono);
873
874 } while (read_seqcount_retry(&tk_core.seq, seq));
875
876 return ktime_add_ns(base, nsecs);
877
878}
879EXPORT_SYMBOL_GPL(ktime_get_with_offset);
880
881ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
882{
883 struct timekeeper *tk = &tk_core.timekeeper;
884 unsigned int seq;
885 ktime_t base, *offset = offsets[offs];
886 u64 nsecs;
887
888 WARN_ON(timekeeping_suspended);
889
890 do {
891 seq = read_seqcount_begin(&tk_core.seq);
892 base = ktime_add(tk->tkr_mono.base, *offset);
893 nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
894
895 } while (read_seqcount_retry(&tk_core.seq, seq));
896
897 return ktime_add_ns(base, nsecs);
898}
899EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
900
901
902
903
904
905
906ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
907{
908 ktime_t *offset = offsets[offs];
909 unsigned int seq;
910 ktime_t tconv;
911
912 do {
913 seq = read_seqcount_begin(&tk_core.seq);
914 tconv = ktime_add(tmono, *offset);
915 } while (read_seqcount_retry(&tk_core.seq, seq));
916
917 return tconv;
918}
919EXPORT_SYMBOL_GPL(ktime_mono_to_any);
920
921
922
923
924ktime_t ktime_get_raw(void)
925{
926 struct timekeeper *tk = &tk_core.timekeeper;
927 unsigned int seq;
928 ktime_t base;
929 u64 nsecs;
930
931 do {
932 seq = read_seqcount_begin(&tk_core.seq);
933 base = tk->tkr_raw.base;
934 nsecs = timekeeping_get_ns(&tk->tkr_raw);
935
936 } while (read_seqcount_retry(&tk_core.seq, seq));
937
938 return ktime_add_ns(base, nsecs);
939}
940EXPORT_SYMBOL_GPL(ktime_get_raw);
941
942
943
944
945
946
947
948
949
950void ktime_get_ts64(struct timespec64 *ts)
951{
952 struct timekeeper *tk = &tk_core.timekeeper;
953 struct timespec64 tomono;
954 unsigned int seq;
955 u64 nsec;
956
957 WARN_ON(timekeeping_suspended);
958
959 do {
960 seq = read_seqcount_begin(&tk_core.seq);
961 ts->tv_sec = tk->xtime_sec;
962 nsec = timekeeping_get_ns(&tk->tkr_mono);
963 tomono = tk->wall_to_monotonic;
964
965 } while (read_seqcount_retry(&tk_core.seq, seq));
966
967 ts->tv_sec += tomono.tv_sec;
968 ts->tv_nsec = 0;
969 timespec64_add_ns(ts, nsec + tomono.tv_nsec);
970}
971EXPORT_SYMBOL_GPL(ktime_get_ts64);
972
973
974
975
976
977
978
979
980
981
982time64_t ktime_get_seconds(void)
983{
984 struct timekeeper *tk = &tk_core.timekeeper;
985
986 WARN_ON(timekeeping_suspended);
987 return tk->ktime_sec;
988}
989EXPORT_SYMBOL_GPL(ktime_get_seconds);
990
991
992
993
994
995
996
997
998
999
1000
1001time64_t ktime_get_real_seconds(void)
1002{
1003 struct timekeeper *tk = &tk_core.timekeeper;
1004 time64_t seconds;
1005 unsigned int seq;
1006
1007 if (IS_ENABLED(CONFIG_64BIT))
1008 return tk->xtime_sec;
1009
1010 do {
1011 seq = read_seqcount_begin(&tk_core.seq);
1012 seconds = tk->xtime_sec;
1013
1014 } while (read_seqcount_retry(&tk_core.seq, seq));
1015
1016 return seconds;
1017}
1018EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
1019
1020
1021
1022
1023
1024
1025noinstr time64_t __ktime_get_real_seconds(void)
1026{
1027 struct timekeeper *tk = &tk_core.timekeeper;
1028
1029 return tk->xtime_sec;
1030}
1031
1032
1033
1034
1035
1036void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
1037{
1038 struct timekeeper *tk = &tk_core.timekeeper;
1039 unsigned int seq;
1040 ktime_t base_raw;
1041 ktime_t base_real;
1042 u64 nsec_raw;
1043 u64 nsec_real;
1044 u64 now;
1045
1046 WARN_ON_ONCE(timekeeping_suspended);
1047
1048 do {
1049 seq = read_seqcount_begin(&tk_core.seq);
1050 now = tk_clock_read(&tk->tkr_mono);
1051 systime_snapshot->cs_id = tk->tkr_mono.clock->id;
1052 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
1053 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
1054 base_real = ktime_add(tk->tkr_mono.base,
1055 tk_core.timekeeper.offs_real);
1056 base_raw = tk->tkr_raw.base;
1057 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
1058 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
1059 } while (read_seqcount_retry(&tk_core.seq, seq));
1060
1061 systime_snapshot->cycles = now;
1062 systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
1063 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
1064}
1065EXPORT_SYMBOL_GPL(ktime_get_snapshot);
1066
1067
1068static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
1069{
1070 u64 tmp, rem;
1071
1072 tmp = div64_u64_rem(*base, div, &rem);
1073
1074 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
1075 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
1076 return -EOVERFLOW;
1077 tmp *= mult;
1078
1079 rem = div64_u64(rem * mult, div);
1080 *base = tmp + rem;
1081 return 0;
1082}
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1102 u64 partial_history_cycles,
1103 u64 total_history_cycles,
1104 bool discontinuity,
1105 struct system_device_crosststamp *ts)
1106{
1107 struct timekeeper *tk = &tk_core.timekeeper;
1108 u64 corr_raw, corr_real;
1109 bool interp_forward;
1110 int ret;
1111
1112 if (total_history_cycles == 0 || partial_history_cycles == 0)
1113 return 0;
1114
1115
1116 interp_forward = partial_history_cycles > total_history_cycles / 2;
1117 partial_history_cycles = interp_forward ?
1118 total_history_cycles - partial_history_cycles :
1119 partial_history_cycles;
1120
1121
1122
1123
1124
1125 corr_raw = (u64)ktime_to_ns(
1126 ktime_sub(ts->sys_monoraw, history->raw));
1127 ret = scale64_check_overflow(partial_history_cycles,
1128 total_history_cycles, &corr_raw);
1129 if (ret)
1130 return ret;
1131
1132
1133
1134
1135
1136
1137
1138
1139 if (discontinuity) {
1140 corr_real = mul_u64_u32_div
1141 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
1142 } else {
1143 corr_real = (u64)ktime_to_ns(
1144 ktime_sub(ts->sys_realtime, history->real));
1145 ret = scale64_check_overflow(partial_history_cycles,
1146 total_history_cycles, &corr_real);
1147 if (ret)
1148 return ret;
1149 }
1150
1151
1152 if (interp_forward) {
1153 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1154 ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1155 } else {
1156 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1157 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1158 }
1159
1160 return 0;
1161}
1162
1163
1164
1165
1166static bool cycle_between(u64 before, u64 test, u64 after)
1167{
1168 if (test > before && test < after)
1169 return true;
1170 if (test < before && before > after)
1171 return true;
1172 return false;
1173}
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186int get_device_system_crosststamp(int (*get_time_fn)
1187 (ktime_t *device_time,
1188 struct system_counterval_t *sys_counterval,
1189 void *ctx),
1190 void *ctx,
1191 struct system_time_snapshot *history_begin,
1192 struct system_device_crosststamp *xtstamp)
1193{
1194 struct system_counterval_t system_counterval;
1195 struct timekeeper *tk = &tk_core.timekeeper;
1196 u64 cycles, now, interval_start;
1197 unsigned int clock_was_set_seq = 0;
1198 ktime_t base_real, base_raw;
1199 u64 nsec_real, nsec_raw;
1200 u8 cs_was_changed_seq;
1201 unsigned int seq;
1202 bool do_interp;
1203 int ret;
1204
1205 do {
1206 seq = read_seqcount_begin(&tk_core.seq);
1207
1208
1209
1210
1211 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1212 if (ret)
1213 return ret;
1214
1215
1216
1217
1218
1219
1220 if (tk->tkr_mono.clock != system_counterval.cs)
1221 return -ENODEV;
1222 cycles = system_counterval.cycles;
1223
1224
1225
1226
1227
1228 now = tk_clock_read(&tk->tkr_mono);
1229 interval_start = tk->tkr_mono.cycle_last;
1230 if (!cycle_between(interval_start, cycles, now)) {
1231 clock_was_set_seq = tk->clock_was_set_seq;
1232 cs_was_changed_seq = tk->cs_was_changed_seq;
1233 cycles = interval_start;
1234 do_interp = true;
1235 } else {
1236 do_interp = false;
1237 }
1238
1239 base_real = ktime_add(tk->tkr_mono.base,
1240 tk_core.timekeeper.offs_real);
1241 base_raw = tk->tkr_raw.base;
1242
1243 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1244 system_counterval.cycles);
1245 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1246 system_counterval.cycles);
1247 } while (read_seqcount_retry(&tk_core.seq, seq));
1248
1249 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1250 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1251
1252
1253
1254
1255
1256 if (do_interp) {
1257 u64 partial_history_cycles, total_history_cycles;
1258 bool discontinuity;
1259
1260
1261
1262
1263
1264
1265 if (!history_begin ||
1266 !cycle_between(history_begin->cycles,
1267 system_counterval.cycles, cycles) ||
1268 history_begin->cs_was_changed_seq != cs_was_changed_seq)
1269 return -EINVAL;
1270 partial_history_cycles = cycles - system_counterval.cycles;
1271 total_history_cycles = cycles - history_begin->cycles;
1272 discontinuity =
1273 history_begin->clock_was_set_seq != clock_was_set_seq;
1274
1275 ret = adjust_historical_crosststamp(history_begin,
1276 partial_history_cycles,
1277 total_history_cycles,
1278 discontinuity, xtstamp);
1279 if (ret)
1280 return ret;
1281 }
1282
1283 return 0;
1284}
1285EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1286
1287
1288
1289
1290
1291
1292
1293int do_settimeofday64(const struct timespec64 *ts)
1294{
1295 struct timekeeper *tk = &tk_core.timekeeper;
1296 struct timespec64 ts_delta, xt;
1297 unsigned long flags;
1298 int ret = 0;
1299
1300 if (!timespec64_valid_settod(ts))
1301 return -EINVAL;
1302
1303 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1304 write_seqcount_begin(&tk_core.seq);
1305
1306 timekeeping_forward_now(tk);
1307
1308 xt = tk_xtime(tk);
1309 ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
1310 ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
1311
1312 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
1313 ret = -EINVAL;
1314 goto out;
1315 }
1316
1317 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
1318
1319 tk_set_xtime(tk, ts);
1320out:
1321 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1322
1323 write_seqcount_end(&tk_core.seq);
1324 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1325
1326
1327 clock_was_set(CLOCK_SET_WALL);
1328
1329 if (!ret)
1330 audit_tk_injoffset(ts_delta);
1331
1332 return ret;
1333}
1334EXPORT_SYMBOL(do_settimeofday64);
1335
1336
1337
1338
1339
1340
1341
1342static int timekeeping_inject_offset(const struct timespec64 *ts)
1343{
1344 struct timekeeper *tk = &tk_core.timekeeper;
1345 unsigned long flags;
1346 struct timespec64 tmp;
1347 int ret = 0;
1348
1349 if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
1350 return -EINVAL;
1351
1352 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1353 write_seqcount_begin(&tk_core.seq);
1354
1355 timekeeping_forward_now(tk);
1356
1357
1358 tmp = timespec64_add(tk_xtime(tk), *ts);
1359 if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
1360 !timespec64_valid_settod(&tmp)) {
1361 ret = -EINVAL;
1362 goto error;
1363 }
1364
1365 tk_xtime_add(tk, ts);
1366 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
1367
1368error:
1369 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1370
1371 write_seqcount_end(&tk_core.seq);
1372 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1373
1374
1375 clock_was_set(CLOCK_SET_WALL);
1376
1377 return ret;
1378}
1379
1380
1381
1382
1383
1384int persistent_clock_is_local;
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402void timekeeping_warp_clock(void)
1403{
1404 if (sys_tz.tz_minuteswest != 0) {
1405 struct timespec64 adjust;
1406
1407 persistent_clock_is_local = 1;
1408 adjust.tv_sec = sys_tz.tz_minuteswest * 60;
1409 adjust.tv_nsec = 0;
1410 timekeeping_inject_offset(&adjust);
1411 }
1412}
1413
1414
1415
1416
1417static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1418{
1419 tk->tai_offset = tai_offset;
1420 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1421}
1422
1423
1424
1425
1426
1427
1428static int change_clocksource(void *data)
1429{
1430 struct timekeeper *tk = &tk_core.timekeeper;
1431 struct clocksource *new, *old = NULL;
1432 unsigned long flags;
1433 bool change = false;
1434
1435 new = (struct clocksource *) data;
1436
1437
1438
1439
1440
1441 if (try_module_get(new->owner)) {
1442 if (!new->enable || new->enable(new) == 0)
1443 change = true;
1444 else
1445 module_put(new->owner);
1446 }
1447
1448 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1449 write_seqcount_begin(&tk_core.seq);
1450
1451 timekeeping_forward_now(tk);
1452
1453 if (change) {
1454 old = tk->tkr_mono.clock;
1455 tk_setup_internals(tk, new);
1456 }
1457
1458 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1459
1460 write_seqcount_end(&tk_core.seq);
1461 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1462
1463 if (old) {
1464 if (old->disable)
1465 old->disable(old);
1466
1467 module_put(old->owner);
1468 }
1469
1470 return 0;
1471}
1472
1473
1474
1475
1476
1477
1478
1479
1480int timekeeping_notify(struct clocksource *clock)
1481{
1482 struct timekeeper *tk = &tk_core.timekeeper;
1483
1484 if (tk->tkr_mono.clock == clock)
1485 return 0;
1486 stop_machine(change_clocksource, clock, NULL);
1487 tick_clock_notify();
1488 return tk->tkr_mono.clock == clock ? 0 : -1;
1489}
1490
1491
1492
1493
1494
1495
1496
1497void ktime_get_raw_ts64(struct timespec64 *ts)
1498{
1499 struct timekeeper *tk = &tk_core.timekeeper;
1500 unsigned int seq;
1501 u64 nsecs;
1502
1503 do {
1504 seq = read_seqcount_begin(&tk_core.seq);
1505 ts->tv_sec = tk->raw_sec;
1506 nsecs = timekeeping_get_ns(&tk->tkr_raw);
1507
1508 } while (read_seqcount_retry(&tk_core.seq, seq));
1509
1510 ts->tv_nsec = 0;
1511 timespec64_add_ns(ts, nsecs);
1512}
1513EXPORT_SYMBOL(ktime_get_raw_ts64);
1514
1515
1516
1517
1518
1519int timekeeping_valid_for_hres(void)
1520{
1521 struct timekeeper *tk = &tk_core.timekeeper;
1522 unsigned int seq;
1523 int ret;
1524
1525 do {
1526 seq = read_seqcount_begin(&tk_core.seq);
1527
1528 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1529
1530 } while (read_seqcount_retry(&tk_core.seq, seq));
1531
1532 return ret;
1533}
1534
1535
1536
1537
1538u64 timekeeping_max_deferment(void)
1539{
1540 struct timekeeper *tk = &tk_core.timekeeper;
1541 unsigned int seq;
1542 u64 ret;
1543
1544 do {
1545 seq = read_seqcount_begin(&tk_core.seq);
1546
1547 ret = tk->tkr_mono.clock->max_idle_ns;
1548
1549 } while (read_seqcount_retry(&tk_core.seq, seq));
1550
1551 return ret;
1552}
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564void __weak read_persistent_clock64(struct timespec64 *ts)
1565{
1566 ts->tv_sec = 0;
1567 ts->tv_nsec = 0;
1568}
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583void __weak __init
1584read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1585 struct timespec64 *boot_offset)
1586{
1587 read_persistent_clock64(wall_time);
1588 *boot_offset = ns_to_timespec64(local_clock());
1589}
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604static bool suspend_timing_needed;
1605
1606
1607static bool persistent_clock_exists;
1608
1609
1610
1611
1612void __init timekeeping_init(void)
1613{
1614 struct timespec64 wall_time, boot_offset, wall_to_mono;
1615 struct timekeeper *tk = &tk_core.timekeeper;
1616 struct clocksource *clock;
1617 unsigned long flags;
1618
1619 read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
1620 if (timespec64_valid_settod(&wall_time) &&
1621 timespec64_to_ns(&wall_time) > 0) {
1622 persistent_clock_exists = true;
1623 } else if (timespec64_to_ns(&wall_time) != 0) {
1624 pr_warn("Persistent clock returned invalid value");
1625 wall_time = (struct timespec64){0};
1626 }
1627
1628 if (timespec64_compare(&wall_time, &boot_offset) < 0)
1629 boot_offset = (struct timespec64){0};
1630
1631
1632
1633
1634
1635 wall_to_mono = timespec64_sub(boot_offset, wall_time);
1636
1637 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1638 write_seqcount_begin(&tk_core.seq);
1639 ntp_init();
1640
1641 clock = clocksource_default_clock();
1642 if (clock->enable)
1643 clock->enable(clock);
1644 tk_setup_internals(tk, clock);
1645
1646 tk_set_xtime(tk, &wall_time);
1647 tk->raw_sec = 0;
1648
1649 tk_set_wall_to_mono(tk, wall_to_mono);
1650
1651 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1652
1653 write_seqcount_end(&tk_core.seq);
1654 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1655}
1656
1657
1658static struct timespec64 timekeeping_suspend_time;
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1669 const struct timespec64 *delta)
1670{
1671 if (!timespec64_valid_strict(delta)) {
1672 printk_deferred(KERN_WARNING
1673 "__timekeeping_inject_sleeptime: Invalid "
1674 "sleep delta value!\n");
1675 return;
1676 }
1677 tk_xtime_add(tk, delta);
1678 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1679 tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1680 tk_debug_account_sleep_time(delta);
1681}
1682
1683#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700bool timekeeping_rtc_skipresume(void)
1701{
1702 return !suspend_timing_needed;
1703}
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714bool timekeeping_rtc_skipsuspend(void)
1715{
1716 return persistent_clock_exists;
1717}
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
1731{
1732 struct timekeeper *tk = &tk_core.timekeeper;
1733 unsigned long flags;
1734
1735 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1736 write_seqcount_begin(&tk_core.seq);
1737
1738 suspend_timing_needed = false;
1739
1740 timekeeping_forward_now(tk);
1741
1742 __timekeeping_inject_sleeptime(tk, delta);
1743
1744 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1745
1746 write_seqcount_end(&tk_core.seq);
1747 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1748
1749
1750 clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT);
1751}
1752#endif
1753
1754
1755
1756
1757void timekeeping_resume(void)
1758{
1759 struct timekeeper *tk = &tk_core.timekeeper;
1760 struct clocksource *clock = tk->tkr_mono.clock;
1761 unsigned long flags;
1762 struct timespec64 ts_new, ts_delta;
1763 u64 cycle_now, nsec;
1764 bool inject_sleeptime = false;
1765
1766 read_persistent_clock64(&ts_new);
1767
1768 clockevents_resume();
1769 clocksource_resume();
1770
1771 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1772 write_seqcount_begin(&tk_core.seq);
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786 cycle_now = tk_clock_read(&tk->tkr_mono);
1787 nsec = clocksource_stop_suspend_timing(clock, cycle_now);
1788 if (nsec > 0) {
1789 ts_delta = ns_to_timespec64(nsec);
1790 inject_sleeptime = true;
1791 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1792 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1793 inject_sleeptime = true;
1794 }
1795
1796 if (inject_sleeptime) {
1797 suspend_timing_needed = false;
1798 __timekeeping_inject_sleeptime(tk, &ts_delta);
1799 }
1800
1801
1802 tk->tkr_mono.cycle_last = cycle_now;
1803 tk->tkr_raw.cycle_last = cycle_now;
1804
1805 tk->ntp_error = 0;
1806 timekeeping_suspended = 0;
1807 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1808 write_seqcount_end(&tk_core.seq);
1809 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1810
1811 touch_softlockup_watchdog();
1812
1813
1814 tick_resume();
1815
1816 timerfd_resume();
1817}
1818
1819int timekeeping_suspend(void)
1820{
1821 struct timekeeper *tk = &tk_core.timekeeper;
1822 unsigned long flags;
1823 struct timespec64 delta, delta_delta;
1824 static struct timespec64 old_delta;
1825 struct clocksource *curr_clock;
1826 u64 cycle_now;
1827
1828 read_persistent_clock64(&timekeeping_suspend_time);
1829
1830
1831
1832
1833
1834
1835 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1836 persistent_clock_exists = true;
1837
1838 suspend_timing_needed = true;
1839
1840 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1841 write_seqcount_begin(&tk_core.seq);
1842 timekeeping_forward_now(tk);
1843 timekeeping_suspended = 1;
1844
1845
1846
1847
1848
1849
1850 curr_clock = tk->tkr_mono.clock;
1851 cycle_now = tk->tkr_mono.cycle_last;
1852 clocksource_start_suspend_timing(curr_clock, cycle_now);
1853
1854 if (persistent_clock_exists) {
1855
1856
1857
1858
1859
1860
1861 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1862 delta_delta = timespec64_sub(delta, old_delta);
1863 if (abs(delta_delta.tv_sec) >= 2) {
1864
1865
1866
1867
1868 old_delta = delta;
1869 } else {
1870
1871 timekeeping_suspend_time =
1872 timespec64_add(timekeeping_suspend_time, delta_delta);
1873 }
1874 }
1875
1876 timekeeping_update(tk, TK_MIRROR);
1877 halt_fast_timekeeper(tk);
1878 write_seqcount_end(&tk_core.seq);
1879 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1880
1881 tick_suspend();
1882 clocksource_suspend();
1883 clockevents_suspend();
1884
1885 return 0;
1886}
1887
1888
1889static struct syscore_ops timekeeping_syscore_ops = {
1890 .resume = timekeeping_resume,
1891 .suspend = timekeeping_suspend,
1892};
1893
1894static int __init timekeeping_init_ops(void)
1895{
1896 register_syscore_ops(&timekeeping_syscore_ops);
1897 return 0;
1898}
1899device_initcall(timekeeping_init_ops);
1900
1901
1902
1903
1904static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1905 s64 offset,
1906 s32 mult_adj)
1907{
1908 s64 interval = tk->cycle_interval;
1909
1910 if (mult_adj == 0) {
1911 return;
1912 } else if (mult_adj == -1) {
1913 interval = -interval;
1914 offset = -offset;
1915 } else if (mult_adj != 1) {
1916 interval *= mult_adj;
1917 offset *= mult_adj;
1918 }
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1968
1969 WARN_ON_ONCE(1);
1970 return;
1971 }
1972
1973 tk->tkr_mono.mult += mult_adj;
1974 tk->xtime_interval += interval;
1975 tk->tkr_mono.xtime_nsec -= offset;
1976}
1977
1978
1979
1980
1981
1982static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1983{
1984 u32 mult;
1985
1986
1987
1988
1989
1990 if (likely(tk->ntp_tick == ntp_tick_length())) {
1991 mult = tk->tkr_mono.mult - tk->ntp_err_mult;
1992 } else {
1993 tk->ntp_tick = ntp_tick_length();
1994 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
1995 tk->xtime_remainder, tk->cycle_interval);
1996 }
1997
1998
1999
2000
2001
2002
2003
2004 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
2005 mult += tk->ntp_err_mult;
2006
2007 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
2008
2009 if (unlikely(tk->tkr_mono.clock->maxadj &&
2010 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
2011 > tk->tkr_mono.clock->maxadj))) {
2012 printk_once(KERN_WARNING
2013 "Adjusting %s more than 11%% (%ld vs %ld)\n",
2014 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
2015 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
2016 }
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
2029 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
2030 tk->tkr_mono.shift;
2031 tk->xtime_sec--;
2032 tk->skip_second_overflow = 1;
2033 }
2034}
2035
2036
2037
2038
2039
2040
2041
2042
2043static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
2044{
2045 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
2046 unsigned int clock_set = 0;
2047
2048 while (tk->tkr_mono.xtime_nsec >= nsecps) {
2049 int leap;
2050
2051 tk->tkr_mono.xtime_nsec -= nsecps;
2052 tk->xtime_sec++;
2053
2054
2055
2056
2057
2058 if (unlikely(tk->skip_second_overflow)) {
2059 tk->skip_second_overflow = 0;
2060 continue;
2061 }
2062
2063
2064 leap = second_overflow(tk->xtime_sec);
2065 if (unlikely(leap)) {
2066 struct timespec64 ts;
2067
2068 tk->xtime_sec += leap;
2069
2070 ts.tv_sec = leap;
2071 ts.tv_nsec = 0;
2072 tk_set_wall_to_mono(tk,
2073 timespec64_sub(tk->wall_to_monotonic, ts));
2074
2075 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
2076
2077 clock_set = TK_CLOCK_WAS_SET;
2078 }
2079 }
2080 return clock_set;
2081}
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2093 u32 shift, unsigned int *clock_set)
2094{
2095 u64 interval = tk->cycle_interval << shift;
2096 u64 snsec_per_sec;
2097
2098
2099 if (offset < interval)
2100 return offset;
2101
2102
2103 offset -= interval;
2104 tk->tkr_mono.cycle_last += interval;
2105 tk->tkr_raw.cycle_last += interval;
2106
2107 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2108 *clock_set |= accumulate_nsecs_to_secs(tk);
2109
2110
2111 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2112 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2113 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2114 tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2115 tk->raw_sec++;
2116 }
2117
2118
2119 tk->ntp_error += tk->ntp_tick << shift;
2120 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2121 (tk->ntp_error_shift + shift);
2122
2123 return offset;
2124}
2125
2126
2127
2128
2129
2130static bool timekeeping_advance(enum timekeeping_adv_mode mode)
2131{
2132 struct timekeeper *real_tk = &tk_core.timekeeper;
2133 struct timekeeper *tk = &shadow_timekeeper;
2134 u64 offset;
2135 int shift = 0, maxshift;
2136 unsigned int clock_set = 0;
2137 unsigned long flags;
2138
2139 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2140
2141
2142 if (unlikely(timekeeping_suspended))
2143 goto out;
2144
2145 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
2146 tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
2147
2148
2149 if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
2150 goto out;
2151
2152
2153 timekeeping_check_update(tk, offset);
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163 shift = ilog2(offset) - ilog2(tk->cycle_interval);
2164 shift = max(0, shift);
2165
2166 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
2167 shift = min(shift, maxshift);
2168 while (offset >= tk->cycle_interval) {
2169 offset = logarithmic_accumulation(tk, offset, shift,
2170 &clock_set);
2171 if (offset < tk->cycle_interval<<shift)
2172 shift--;
2173 }
2174
2175
2176 timekeeping_adjust(tk, offset);
2177
2178
2179
2180
2181
2182 clock_set |= accumulate_nsecs_to_secs(tk);
2183
2184 write_seqcount_begin(&tk_core.seq);
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195 timekeeping_update(tk, clock_set);
2196 memcpy(real_tk, tk, sizeof(*tk));
2197
2198 write_seqcount_end(&tk_core.seq);
2199out:
2200 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2201
2202 return !!clock_set;
2203}
2204
2205
2206
2207
2208
2209void update_wall_time(void)
2210{
2211 if (timekeeping_advance(TK_ADV_TICK))
2212 clock_was_set_delayed();
2213}
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226void getboottime64(struct timespec64 *ts)
2227{
2228 struct timekeeper *tk = &tk_core.timekeeper;
2229 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2230
2231 *ts = ktime_to_timespec64(t);
2232}
2233EXPORT_SYMBOL_GPL(getboottime64);
2234
2235void ktime_get_coarse_real_ts64(struct timespec64 *ts)
2236{
2237 struct timekeeper *tk = &tk_core.timekeeper;
2238 unsigned int seq;
2239
2240 do {
2241 seq = read_seqcount_begin(&tk_core.seq);
2242
2243 *ts = tk_xtime(tk);
2244 } while (read_seqcount_retry(&tk_core.seq, seq));
2245}
2246EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
2247
2248void ktime_get_coarse_ts64(struct timespec64 *ts)
2249{
2250 struct timekeeper *tk = &tk_core.timekeeper;
2251 struct timespec64 now, mono;
2252 unsigned int seq;
2253
2254 do {
2255 seq = read_seqcount_begin(&tk_core.seq);
2256
2257 now = tk_xtime(tk);
2258 mono = tk->wall_to_monotonic;
2259 } while (read_seqcount_retry(&tk_core.seq, seq));
2260
2261 set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
2262 now.tv_nsec + mono.tv_nsec);
2263}
2264EXPORT_SYMBOL(ktime_get_coarse_ts64);
2265
2266
2267
2268
2269void do_timer(unsigned long ticks)
2270{
2271 jiffies_64 += ticks;
2272 calc_global_load();
2273}
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2289 ktime_t *offs_boot, ktime_t *offs_tai)
2290{
2291 struct timekeeper *tk = &tk_core.timekeeper;
2292 unsigned int seq;
2293 ktime_t base;
2294 u64 nsecs;
2295
2296 do {
2297 seq = read_seqcount_begin(&tk_core.seq);
2298
2299 base = tk->tkr_mono.base;
2300 nsecs = timekeeping_get_ns(&tk->tkr_mono);
2301 base = ktime_add_ns(base, nsecs);
2302
2303 if (*cwsseq != tk->clock_was_set_seq) {
2304 *cwsseq = tk->clock_was_set_seq;
2305 *offs_real = tk->offs_real;
2306 *offs_boot = tk->offs_boot;
2307 *offs_tai = tk->offs_tai;
2308 }
2309
2310
2311 if (unlikely(base >= tk->next_leap_ktime))
2312 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2313
2314 } while (read_seqcount_retry(&tk_core.seq, seq));
2315
2316 return base;
2317}
2318
2319
2320
2321
2322static int timekeeping_validate_timex(const struct __kernel_timex *txc)
2323{
2324 if (txc->modes & ADJ_ADJTIME) {
2325
2326 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2327 return -EINVAL;
2328 if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2329 !capable(CAP_SYS_TIME))
2330 return -EPERM;
2331 } else {
2332
2333 if (txc->modes && !capable(CAP_SYS_TIME))
2334 return -EPERM;
2335
2336
2337
2338
2339 if (txc->modes & ADJ_TICK &&
2340 (txc->tick < 900000/USER_HZ ||
2341 txc->tick > 1100000/USER_HZ))
2342 return -EINVAL;
2343 }
2344
2345 if (txc->modes & ADJ_SETOFFSET) {
2346
2347 if (!capable(CAP_SYS_TIME))
2348 return -EPERM;
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358 if (txc->time.tv_usec < 0)
2359 return -EINVAL;
2360
2361 if (txc->modes & ADJ_NANO) {
2362 if (txc->time.tv_usec >= NSEC_PER_SEC)
2363 return -EINVAL;
2364 } else {
2365 if (txc->time.tv_usec >= USEC_PER_SEC)
2366 return -EINVAL;
2367 }
2368 }
2369
2370
2371
2372
2373
2374 if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
2375 if (LLONG_MIN / PPM_SCALE > txc->freq)
2376 return -EINVAL;
2377 if (LLONG_MAX / PPM_SCALE < txc->freq)
2378 return -EINVAL;
2379 }
2380
2381 return 0;
2382}
2383
2384
2385
2386
2387
2388int do_adjtimex(struct __kernel_timex *txc)
2389{
2390 struct timekeeper *tk = &tk_core.timekeeper;
2391 struct audit_ntp_data ad;
2392 bool clock_set = false;
2393 struct timespec64 ts;
2394 unsigned long flags;
2395 s32 orig_tai, tai;
2396 int ret;
2397
2398
2399 ret = timekeeping_validate_timex(txc);
2400 if (ret)
2401 return ret;
2402
2403 if (txc->modes & ADJ_SETOFFSET) {
2404 struct timespec64 delta;
2405 delta.tv_sec = txc->time.tv_sec;
2406 delta.tv_nsec = txc->time.tv_usec;
2407 if (!(txc->modes & ADJ_NANO))
2408 delta.tv_nsec *= 1000;
2409 ret = timekeeping_inject_offset(&delta);
2410 if (ret)
2411 return ret;
2412
2413 audit_tk_injoffset(delta);
2414 }
2415
2416 audit_ntp_init(&ad);
2417
2418 ktime_get_real_ts64(&ts);
2419
2420 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2421 write_seqcount_begin(&tk_core.seq);
2422
2423 orig_tai = tai = tk->tai_offset;
2424 ret = __do_adjtimex(txc, &ts, &tai, &ad);
2425
2426 if (tai != orig_tai) {
2427 __timekeeping_set_tai_offset(tk, tai);
2428 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2429 clock_set = true;
2430 }
2431 tk_update_leap_state(tk);
2432
2433 write_seqcount_end(&tk_core.seq);
2434 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2435
2436 audit_ntp_log(&ad);
2437
2438
2439 if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
2440 clock_set |= timekeeping_advance(TK_ADV_FREQ);
2441
2442 if (clock_set)
2443 clock_was_set(CLOCK_REALTIME);
2444
2445 ntp_notify_cmos_timer();
2446
2447 return ret;
2448}
2449
2450#ifdef CONFIG_NTP_PPS
2451
2452
2453
2454void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2455{
2456 unsigned long flags;
2457
2458 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2459 write_seqcount_begin(&tk_core.seq);
2460
2461 __hardpps(phase_ts, raw_ts);
2462
2463 write_seqcount_end(&tk_core.seq);
2464 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2465}
2466EXPORT_SYMBOL(hardpps);
2467#endif
2468