1
2
3
4
5
6#include <linux/timekeeper_internal.h>
7#include <linux/module.h>
8#include <linux/interrupt.h>
9#include <linux/percpu.h>
10#include <linux/init.h>
11#include <linux/mm.h>
12#include <linux/nmi.h>
13#include <linux/sched.h>
14#include <linux/sched/loadavg.h>
15#include <linux/sched/clock.h>
16#include <linux/syscore_ops.h>
17#include <linux/clocksource.h>
18#include <linux/jiffies.h>
19#include <linux/time.h>
20#include <linux/tick.h>
21#include <linux/stop_machine.h>
22#include <linux/pvclock_gtod.h>
23#include <linux/compiler.h>
24#include <linux/audit.h>
25
26#include "tick-internal.h"
27#include "ntp_internal.h"
28#include "timekeeping_internal.h"
29
30#define TK_CLEAR_NTP (1 << 0)
31#define TK_MIRROR (1 << 1)
32#define TK_CLOCK_WAS_SET (1 << 2)
33
34enum timekeeping_adv_mode {
35
36 TK_ADV_TICK,
37
38
39 TK_ADV_FREQ
40};
41
42
43
44
45
46static struct {
47 seqcount_t seq;
48 struct timekeeper timekeeper;
49} tk_core ____cacheline_aligned = {
50 .seq = SEQCNT_ZERO(tk_core.seq),
51};
52
53static DEFINE_RAW_SPINLOCK(timekeeper_lock);
54static struct timekeeper shadow_timekeeper;
55
56
57
58
59
60
61
62
63
64
65struct tk_fast {
66 seqcount_t seq;
67 struct tk_read_base base[2];
68};
69
70
71static u64 cycles_at_suspend;
72
73static u64 dummy_clock_read(struct clocksource *cs)
74{
75 return cycles_at_suspend;
76}
77
78static struct clocksource dummy_clock = {
79 .read = dummy_clock_read,
80};
81
82static struct tk_fast tk_fast_mono ____cacheline_aligned = {
83 .base[0] = { .clock = &dummy_clock, },
84 .base[1] = { .clock = &dummy_clock, },
85};
86
87static struct tk_fast tk_fast_raw ____cacheline_aligned = {
88 .base[0] = { .clock = &dummy_clock, },
89 .base[1] = { .clock = &dummy_clock, },
90};
91
92
93int __read_mostly timekeeping_suspended;
94
95static inline void tk_normalize_xtime(struct timekeeper *tk)
96{
97 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
98 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
99 tk->xtime_sec++;
100 }
101 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
102 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
103 tk->raw_sec++;
104 }
105}
106
107static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
108{
109 struct timespec64 ts;
110
111 ts.tv_sec = tk->xtime_sec;
112 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
113 return ts;
114}
115
116static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
117{
118 tk->xtime_sec = ts->tv_sec;
119 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
120}
121
122static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
123{
124 tk->xtime_sec += ts->tv_sec;
125 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
126 tk_normalize_xtime(tk);
127}
128
129static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
130{
131 struct timespec64 tmp;
132
133
134
135
136
137 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
138 -tk->wall_to_monotonic.tv_nsec);
139 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
140 tk->wall_to_monotonic = wtm;
141 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
142 tk->offs_real = timespec64_to_ktime(tmp);
143 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
144}
145
146static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
147{
148 tk->offs_boot = ktime_add(tk->offs_boot, delta);
149}
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164static inline u64 tk_clock_read(const struct tk_read_base *tkr)
165{
166 struct clocksource *clock = READ_ONCE(tkr->clock);
167
168 return clock->read(clock);
169}
170
171#ifdef CONFIG_DEBUG_TIMEKEEPING
172#define WARNING_FREQ (HZ*300)
173
174static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
175{
176
177 u64 max_cycles = tk->tkr_mono.clock->max_cycles;
178 const char *name = tk->tkr_mono.clock->name;
179
180 if (offset > max_cycles) {
181 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
182 offset, name, max_cycles);
183 printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
184 } else {
185 if (offset > (max_cycles >> 1)) {
186 printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
187 offset, name, max_cycles >> 1);
188 printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
189 }
190 }
191
192 if (tk->underflow_seen) {
193 if (jiffies - tk->last_warning > WARNING_FREQ) {
194 printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
195 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
196 printk_deferred(" Your kernel is probably still fine.\n");
197 tk->last_warning = jiffies;
198 }
199 tk->underflow_seen = 0;
200 }
201
202 if (tk->overflow_seen) {
203 if (jiffies - tk->last_warning > WARNING_FREQ) {
204 printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
205 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
206 printk_deferred(" Your kernel is probably still fine.\n");
207 tk->last_warning = jiffies;
208 }
209 tk->overflow_seen = 0;
210 }
211}
212
213static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
214{
215 struct timekeeper *tk = &tk_core.timekeeper;
216 u64 now, last, mask, max, delta;
217 unsigned int seq;
218
219
220
221
222
223
224
225
226 do {
227 seq = read_seqcount_begin(&tk_core.seq);
228 now = tk_clock_read(tkr);
229 last = tkr->cycle_last;
230 mask = tkr->mask;
231 max = tkr->clock->max_cycles;
232 } while (read_seqcount_retry(&tk_core.seq, seq));
233
234 delta = clocksource_delta(now, last, mask);
235
236
237
238
239
240 if (unlikely((~delta & mask) < (mask >> 3))) {
241 tk->underflow_seen = 1;
242 delta = 0;
243 }
244
245
246 if (unlikely(delta > max)) {
247 tk->overflow_seen = 1;
248 delta = tkr->clock->max_cycles;
249 }
250
251 return delta;
252}
253#else
254static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
255{
256}
257static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
258{
259 u64 cycle_now, delta;
260
261
262 cycle_now = tk_clock_read(tkr);
263
264
265 delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
266
267 return delta;
268}
269#endif
270
271
272
273
274
275
276
277
278
279
280
281
282static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
283{
284 u64 interval;
285 u64 tmp, ntpinterval;
286 struct clocksource *old_clock;
287
288 ++tk->cs_was_changed_seq;
289 old_clock = tk->tkr_mono.clock;
290 tk->tkr_mono.clock = clock;
291 tk->tkr_mono.mask = clock->mask;
292 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
293
294 tk->tkr_raw.clock = clock;
295 tk->tkr_raw.mask = clock->mask;
296 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
297
298
299 tmp = NTP_INTERVAL_LENGTH;
300 tmp <<= clock->shift;
301 ntpinterval = tmp;
302 tmp += clock->mult/2;
303 do_div(tmp, clock->mult);
304 if (tmp == 0)
305 tmp = 1;
306
307 interval = (u64) tmp;
308 tk->cycle_interval = interval;
309
310
311 tk->xtime_interval = interval * clock->mult;
312 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
313 tk->raw_interval = interval * clock->mult;
314
315
316 if (old_clock) {
317 int shift_change = clock->shift - old_clock->shift;
318 if (shift_change < 0) {
319 tk->tkr_mono.xtime_nsec >>= -shift_change;
320 tk->tkr_raw.xtime_nsec >>= -shift_change;
321 } else {
322 tk->tkr_mono.xtime_nsec <<= shift_change;
323 tk->tkr_raw.xtime_nsec <<= shift_change;
324 }
325 }
326
327 tk->tkr_mono.shift = clock->shift;
328 tk->tkr_raw.shift = clock->shift;
329
330 tk->ntp_error = 0;
331 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
332 tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
333
334
335
336
337
338
339 tk->tkr_mono.mult = clock->mult;
340 tk->tkr_raw.mult = clock->mult;
341 tk->ntp_err_mult = 0;
342 tk->skip_second_overflow = 0;
343}
344
345
346
347#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
348static u32 default_arch_gettimeoffset(void) { return 0; }
349u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
350#else
351static inline u32 arch_gettimeoffset(void) { return 0; }
352#endif
353
354static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
355{
356 u64 nsec;
357
358 nsec = delta * tkr->mult + tkr->xtime_nsec;
359 nsec >>= tkr->shift;
360
361
362 return nsec + arch_gettimeoffset();
363}
364
365static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
366{
367 u64 delta;
368
369 delta = timekeeping_get_delta(tkr);
370 return timekeeping_delta_to_ns(tkr, delta);
371}
372
373static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
374{
375 u64 delta;
376
377
378 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
379 return timekeeping_delta_to_ns(tkr, delta);
380}
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396static void update_fast_timekeeper(const struct tk_read_base *tkr,
397 struct tk_fast *tkf)
398{
399 struct tk_read_base *base = tkf->base;
400
401
402 raw_write_seqcount_latch(&tkf->seq);
403
404
405 memcpy(base, tkr, sizeof(*base));
406
407
408 raw_write_seqcount_latch(&tkf->seq);
409
410
411 memcpy(base + 1, base, sizeof(*base));
412}
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
447{
448 struct tk_read_base *tkr;
449 unsigned int seq;
450 u64 now;
451
452 do {
453 seq = raw_read_seqcount_latch(&tkf->seq);
454 tkr = tkf->base + (seq & 0x01);
455 now = ktime_to_ns(tkr->base);
456
457 now += timekeeping_delta_to_ns(tkr,
458 clocksource_delta(
459 tk_clock_read(tkr),
460 tkr->cycle_last,
461 tkr->mask));
462 } while (read_seqcount_retry(&tkf->seq, seq));
463
464 return now;
465}
466
467u64 ktime_get_mono_fast_ns(void)
468{
469 return __ktime_get_fast_ns(&tk_fast_mono);
470}
471EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
472
473u64 ktime_get_raw_fast_ns(void)
474{
475 return __ktime_get_fast_ns(&tk_fast_raw);
476}
477EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500u64 notrace ktime_get_boot_fast_ns(void)
501{
502 struct timekeeper *tk = &tk_core.timekeeper;
503
504 return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
505}
506EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
507
508
509
510
511
512static __always_inline u64 __ktime_get_real_fast_ns(struct tk_fast *tkf)
513{
514 struct tk_read_base *tkr;
515 unsigned int seq;
516 u64 now;
517
518 do {
519 seq = raw_read_seqcount_latch(&tkf->seq);
520 tkr = tkf->base + (seq & 0x01);
521 now = ktime_to_ns(tkr->base_real);
522
523 now += timekeeping_delta_to_ns(tkr,
524 clocksource_delta(
525 tk_clock_read(tkr),
526 tkr->cycle_last,
527 tkr->mask));
528 } while (read_seqcount_retry(&tkf->seq, seq));
529
530 return now;
531}
532
533
534
535
536u64 ktime_get_real_fast_ns(void)
537{
538 return __ktime_get_real_fast_ns(&tk_fast_mono);
539}
540EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
541
542
543
544
545
546
547
548
549
550
551
552static void halt_fast_timekeeper(const struct timekeeper *tk)
553{
554 static struct tk_read_base tkr_dummy;
555 const struct tk_read_base *tkr = &tk->tkr_mono;
556
557 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
558 cycles_at_suspend = tk_clock_read(tkr);
559 tkr_dummy.clock = &dummy_clock;
560 tkr_dummy.base_real = tkr->base + tk->offs_real;
561 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
562
563 tkr = &tk->tkr_raw;
564 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
565 tkr_dummy.clock = &dummy_clock;
566 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
567}
568
569static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
570
571static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
572{
573 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
574}
575
576
577
578
579int pvclock_gtod_register_notifier(struct notifier_block *nb)
580{
581 struct timekeeper *tk = &tk_core.timekeeper;
582 unsigned long flags;
583 int ret;
584
585 raw_spin_lock_irqsave(&timekeeper_lock, flags);
586 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
587 update_pvclock_gtod(tk, true);
588 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
589
590 return ret;
591}
592EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
593
594
595
596
597
598int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
599{
600 unsigned long flags;
601 int ret;
602
603 raw_spin_lock_irqsave(&timekeeper_lock, flags);
604 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
605 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
606
607 return ret;
608}
609EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
610
611
612
613
614static inline void tk_update_leap_state(struct timekeeper *tk)
615{
616 tk->next_leap_ktime = ntp_get_next_leap();
617 if (tk->next_leap_ktime != KTIME_MAX)
618
619 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
620}
621
622
623
624
625static inline void tk_update_ktime_data(struct timekeeper *tk)
626{
627 u64 seconds;
628 u32 nsec;
629
630
631
632
633
634
635
636
637 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
638 nsec = (u32) tk->wall_to_monotonic.tv_nsec;
639 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
640
641
642
643
644
645
646 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
647 if (nsec >= NSEC_PER_SEC)
648 seconds++;
649 tk->ktime_sec = seconds;
650
651
652 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
653}
654
655
656static void timekeeping_update(struct timekeeper *tk, unsigned int action)
657{
658 if (action & TK_CLEAR_NTP) {
659 tk->ntp_error = 0;
660 ntp_clear();
661 }
662
663 tk_update_leap_state(tk);
664 tk_update_ktime_data(tk);
665
666 update_vsyscall(tk);
667 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
668
669 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
670 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
671 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
672
673 if (action & TK_CLOCK_WAS_SET)
674 tk->clock_was_set_seq++;
675
676
677
678
679
680 if (action & TK_MIRROR)
681 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
682 sizeof(tk_core.timekeeper));
683}
684
685
686
687
688
689
690
691
692static void timekeeping_forward_now(struct timekeeper *tk)
693{
694 u64 cycle_now, delta;
695
696 cycle_now = tk_clock_read(&tk->tkr_mono);
697 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
698 tk->tkr_mono.cycle_last = cycle_now;
699 tk->tkr_raw.cycle_last = cycle_now;
700
701 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
702
703
704 tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift;
705
706
707 tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
708
709
710 tk->tkr_raw.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_raw.shift;
711
712 tk_normalize_xtime(tk);
713}
714
715
716
717
718
719
720
721void ktime_get_real_ts64(struct timespec64 *ts)
722{
723 struct timekeeper *tk = &tk_core.timekeeper;
724 unsigned int seq;
725 u64 nsecs;
726
727 WARN_ON(timekeeping_suspended);
728
729 do {
730 seq = read_seqcount_begin(&tk_core.seq);
731
732 ts->tv_sec = tk->xtime_sec;
733 nsecs = timekeeping_get_ns(&tk->tkr_mono);
734
735 } while (read_seqcount_retry(&tk_core.seq, seq));
736
737 ts->tv_nsec = 0;
738 timespec64_add_ns(ts, nsecs);
739}
740EXPORT_SYMBOL(ktime_get_real_ts64);
741
742ktime_t ktime_get(void)
743{
744 struct timekeeper *tk = &tk_core.timekeeper;
745 unsigned int seq;
746 ktime_t base;
747 u64 nsecs;
748
749 WARN_ON(timekeeping_suspended);
750
751 do {
752 seq = read_seqcount_begin(&tk_core.seq);
753 base = tk->tkr_mono.base;
754 nsecs = timekeeping_get_ns(&tk->tkr_mono);
755
756 } while (read_seqcount_retry(&tk_core.seq, seq));
757
758 return ktime_add_ns(base, nsecs);
759}
760EXPORT_SYMBOL_GPL(ktime_get);
761
762u32 ktime_get_resolution_ns(void)
763{
764 struct timekeeper *tk = &tk_core.timekeeper;
765 unsigned int seq;
766 u32 nsecs;
767
768 WARN_ON(timekeeping_suspended);
769
770 do {
771 seq = read_seqcount_begin(&tk_core.seq);
772 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
773 } while (read_seqcount_retry(&tk_core.seq, seq));
774
775 return nsecs;
776}
777EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
778
779static ktime_t *offsets[TK_OFFS_MAX] = {
780 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
781 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
782 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
783};
784
785ktime_t ktime_get_with_offset(enum tk_offsets offs)
786{
787 struct timekeeper *tk = &tk_core.timekeeper;
788 unsigned int seq;
789 ktime_t base, *offset = offsets[offs];
790 u64 nsecs;
791
792 WARN_ON(timekeeping_suspended);
793
794 do {
795 seq = read_seqcount_begin(&tk_core.seq);
796 base = ktime_add(tk->tkr_mono.base, *offset);
797 nsecs = timekeeping_get_ns(&tk->tkr_mono);
798
799 } while (read_seqcount_retry(&tk_core.seq, seq));
800
801 return ktime_add_ns(base, nsecs);
802
803}
804EXPORT_SYMBOL_GPL(ktime_get_with_offset);
805
806ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
807{
808 struct timekeeper *tk = &tk_core.timekeeper;
809 unsigned int seq;
810 ktime_t base, *offset = offsets[offs];
811 u64 nsecs;
812
813 WARN_ON(timekeeping_suspended);
814
815 do {
816 seq = read_seqcount_begin(&tk_core.seq);
817 base = ktime_add(tk->tkr_mono.base, *offset);
818 nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
819
820 } while (read_seqcount_retry(&tk_core.seq, seq));
821
822 return base + nsecs;
823}
824EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
825
826
827
828
829
830
831ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
832{
833 ktime_t *offset = offsets[offs];
834 unsigned int seq;
835 ktime_t tconv;
836
837 do {
838 seq = read_seqcount_begin(&tk_core.seq);
839 tconv = ktime_add(tmono, *offset);
840 } while (read_seqcount_retry(&tk_core.seq, seq));
841
842 return tconv;
843}
844EXPORT_SYMBOL_GPL(ktime_mono_to_any);
845
846
847
848
849ktime_t ktime_get_raw(void)
850{
851 struct timekeeper *tk = &tk_core.timekeeper;
852 unsigned int seq;
853 ktime_t base;
854 u64 nsecs;
855
856 do {
857 seq = read_seqcount_begin(&tk_core.seq);
858 base = tk->tkr_raw.base;
859 nsecs = timekeeping_get_ns(&tk->tkr_raw);
860
861 } while (read_seqcount_retry(&tk_core.seq, seq));
862
863 return ktime_add_ns(base, nsecs);
864}
865EXPORT_SYMBOL_GPL(ktime_get_raw);
866
867
868
869
870
871
872
873
874
875void ktime_get_ts64(struct timespec64 *ts)
876{
877 struct timekeeper *tk = &tk_core.timekeeper;
878 struct timespec64 tomono;
879 unsigned int seq;
880 u64 nsec;
881
882 WARN_ON(timekeeping_suspended);
883
884 do {
885 seq = read_seqcount_begin(&tk_core.seq);
886 ts->tv_sec = tk->xtime_sec;
887 nsec = timekeeping_get_ns(&tk->tkr_mono);
888 tomono = tk->wall_to_monotonic;
889
890 } while (read_seqcount_retry(&tk_core.seq, seq));
891
892 ts->tv_sec += tomono.tv_sec;
893 ts->tv_nsec = 0;
894 timespec64_add_ns(ts, nsec + tomono.tv_nsec);
895}
896EXPORT_SYMBOL_GPL(ktime_get_ts64);
897
898
899
900
901
902
903
904
905
906
907time64_t ktime_get_seconds(void)
908{
909 struct timekeeper *tk = &tk_core.timekeeper;
910
911 WARN_ON(timekeeping_suspended);
912 return tk->ktime_sec;
913}
914EXPORT_SYMBOL_GPL(ktime_get_seconds);
915
916
917
918
919
920
921
922
923
924
925
926
927time64_t ktime_get_real_seconds(void)
928{
929 struct timekeeper *tk = &tk_core.timekeeper;
930 time64_t seconds;
931 unsigned int seq;
932
933 if (IS_ENABLED(CONFIG_64BIT))
934 return tk->xtime_sec;
935
936 do {
937 seq = read_seqcount_begin(&tk_core.seq);
938 seconds = tk->xtime_sec;
939
940 } while (read_seqcount_retry(&tk_core.seq, seq));
941
942 return seconds;
943}
944EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
945
946
947
948
949
950
951time64_t __ktime_get_real_seconds(void)
952{
953 struct timekeeper *tk = &tk_core.timekeeper;
954
955 return tk->xtime_sec;
956}
957
958
959
960
961
962void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
963{
964 struct timekeeper *tk = &tk_core.timekeeper;
965 unsigned int seq;
966 ktime_t base_raw;
967 ktime_t base_real;
968 u64 nsec_raw;
969 u64 nsec_real;
970 u64 now;
971
972 WARN_ON_ONCE(timekeeping_suspended);
973
974 do {
975 seq = read_seqcount_begin(&tk_core.seq);
976 now = tk_clock_read(&tk->tkr_mono);
977 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
978 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
979 base_real = ktime_add(tk->tkr_mono.base,
980 tk_core.timekeeper.offs_real);
981 base_raw = tk->tkr_raw.base;
982 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
983 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
984 } while (read_seqcount_retry(&tk_core.seq, seq));
985
986 systime_snapshot->cycles = now;
987 systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
988 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
989}
990EXPORT_SYMBOL_GPL(ktime_get_snapshot);
991
992
993static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
994{
995 u64 tmp, rem;
996
997 tmp = div64_u64_rem(*base, div, &rem);
998
999 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
1000 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
1001 return -EOVERFLOW;
1002 tmp *= mult;
1003 rem *= mult;
1004
1005 do_div(rem, div);
1006 *base = tmp + rem;
1007 return 0;
1008}
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1028 u64 partial_history_cycles,
1029 u64 total_history_cycles,
1030 bool discontinuity,
1031 struct system_device_crosststamp *ts)
1032{
1033 struct timekeeper *tk = &tk_core.timekeeper;
1034 u64 corr_raw, corr_real;
1035 bool interp_forward;
1036 int ret;
1037
1038 if (total_history_cycles == 0 || partial_history_cycles == 0)
1039 return 0;
1040
1041
1042 interp_forward = partial_history_cycles > total_history_cycles / 2;
1043 partial_history_cycles = interp_forward ?
1044 total_history_cycles - partial_history_cycles :
1045 partial_history_cycles;
1046
1047
1048
1049
1050
1051 corr_raw = (u64)ktime_to_ns(
1052 ktime_sub(ts->sys_monoraw, history->raw));
1053 ret = scale64_check_overflow(partial_history_cycles,
1054 total_history_cycles, &corr_raw);
1055 if (ret)
1056 return ret;
1057
1058
1059
1060
1061
1062
1063
1064
1065 if (discontinuity) {
1066 corr_real = mul_u64_u32_div
1067 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
1068 } else {
1069 corr_real = (u64)ktime_to_ns(
1070 ktime_sub(ts->sys_realtime, history->real));
1071 ret = scale64_check_overflow(partial_history_cycles,
1072 total_history_cycles, &corr_real);
1073 if (ret)
1074 return ret;
1075 }
1076
1077
1078 if (interp_forward) {
1079 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1080 ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1081 } else {
1082 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1083 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1084 }
1085
1086 return 0;
1087}
1088
1089
1090
1091
1092static bool cycle_between(u64 before, u64 test, u64 after)
1093{
1094 if (test > before && test < after)
1095 return true;
1096 if (test < before && before > after)
1097 return true;
1098 return false;
1099}
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112int get_device_system_crosststamp(int (*get_time_fn)
1113 (ktime_t *device_time,
1114 struct system_counterval_t *sys_counterval,
1115 void *ctx),
1116 void *ctx,
1117 struct system_time_snapshot *history_begin,
1118 struct system_device_crosststamp *xtstamp)
1119{
1120 struct system_counterval_t system_counterval;
1121 struct timekeeper *tk = &tk_core.timekeeper;
1122 u64 cycles, now, interval_start;
1123 unsigned int clock_was_set_seq = 0;
1124 ktime_t base_real, base_raw;
1125 u64 nsec_real, nsec_raw;
1126 u8 cs_was_changed_seq;
1127 unsigned int seq;
1128 bool do_interp;
1129 int ret;
1130
1131 do {
1132 seq = read_seqcount_begin(&tk_core.seq);
1133
1134
1135
1136
1137 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1138 if (ret)
1139 return ret;
1140
1141
1142
1143
1144
1145
1146 if (tk->tkr_mono.clock != system_counterval.cs)
1147 return -ENODEV;
1148 cycles = system_counterval.cycles;
1149
1150
1151
1152
1153
1154 now = tk_clock_read(&tk->tkr_mono);
1155 interval_start = tk->tkr_mono.cycle_last;
1156 if (!cycle_between(interval_start, cycles, now)) {
1157 clock_was_set_seq = tk->clock_was_set_seq;
1158 cs_was_changed_seq = tk->cs_was_changed_seq;
1159 cycles = interval_start;
1160 do_interp = true;
1161 } else {
1162 do_interp = false;
1163 }
1164
1165 base_real = ktime_add(tk->tkr_mono.base,
1166 tk_core.timekeeper.offs_real);
1167 base_raw = tk->tkr_raw.base;
1168
1169 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1170 system_counterval.cycles);
1171 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1172 system_counterval.cycles);
1173 } while (read_seqcount_retry(&tk_core.seq, seq));
1174
1175 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1176 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1177
1178
1179
1180
1181
1182 if (do_interp) {
1183 u64 partial_history_cycles, total_history_cycles;
1184 bool discontinuity;
1185
1186
1187
1188
1189
1190
1191 if (!history_begin ||
1192 !cycle_between(history_begin->cycles,
1193 system_counterval.cycles, cycles) ||
1194 history_begin->cs_was_changed_seq != cs_was_changed_seq)
1195 return -EINVAL;
1196 partial_history_cycles = cycles - system_counterval.cycles;
1197 total_history_cycles = cycles - history_begin->cycles;
1198 discontinuity =
1199 history_begin->clock_was_set_seq != clock_was_set_seq;
1200
1201 ret = adjust_historical_crosststamp(history_begin,
1202 partial_history_cycles,
1203 total_history_cycles,
1204 discontinuity, xtstamp);
1205 if (ret)
1206 return ret;
1207 }
1208
1209 return 0;
1210}
1211EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1212
1213
1214
1215
1216
1217
1218
1219int do_settimeofday64(const struct timespec64 *ts)
1220{
1221 struct timekeeper *tk = &tk_core.timekeeper;
1222 struct timespec64 ts_delta, xt;
1223 unsigned long flags;
1224 int ret = 0;
1225
1226 if (!timespec64_valid_settod(ts))
1227 return -EINVAL;
1228
1229 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1230 write_seqcount_begin(&tk_core.seq);
1231
1232 timekeeping_forward_now(tk);
1233
1234 xt = tk_xtime(tk);
1235 ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
1236 ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
1237
1238 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
1239 ret = -EINVAL;
1240 goto out;
1241 }
1242
1243 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
1244
1245 tk_set_xtime(tk, ts);
1246out:
1247 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1248
1249 write_seqcount_end(&tk_core.seq);
1250 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1251
1252
1253 clock_was_set();
1254
1255 if (!ret)
1256 audit_tk_injoffset(ts_delta);
1257
1258 return ret;
1259}
1260EXPORT_SYMBOL(do_settimeofday64);
1261
1262
1263
1264
1265
1266
1267
1268static int timekeeping_inject_offset(const struct timespec64 *ts)
1269{
1270 struct timekeeper *tk = &tk_core.timekeeper;
1271 unsigned long flags;
1272 struct timespec64 tmp;
1273 int ret = 0;
1274
1275 if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
1276 return -EINVAL;
1277
1278 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1279 write_seqcount_begin(&tk_core.seq);
1280
1281 timekeeping_forward_now(tk);
1282
1283
1284 tmp = timespec64_add(tk_xtime(tk), *ts);
1285 if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
1286 !timespec64_valid_settod(&tmp)) {
1287 ret = -EINVAL;
1288 goto error;
1289 }
1290
1291 tk_xtime_add(tk, ts);
1292 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
1293
1294error:
1295 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1296
1297 write_seqcount_end(&tk_core.seq);
1298 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1299
1300
1301 clock_was_set();
1302
1303 return ret;
1304}
1305
1306
1307
1308
1309
1310int persistent_clock_is_local;
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328void timekeeping_warp_clock(void)
1329{
1330 if (sys_tz.tz_minuteswest != 0) {
1331 struct timespec64 adjust;
1332
1333 persistent_clock_is_local = 1;
1334 adjust.tv_sec = sys_tz.tz_minuteswest * 60;
1335 adjust.tv_nsec = 0;
1336 timekeeping_inject_offset(&adjust);
1337 }
1338}
1339
1340
1341
1342
1343
1344static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1345{
1346 tk->tai_offset = tai_offset;
1347 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1348}
1349
1350
1351
1352
1353
1354
1355static int change_clocksource(void *data)
1356{
1357 struct timekeeper *tk = &tk_core.timekeeper;
1358 struct clocksource *new, *old;
1359 unsigned long flags;
1360
1361 new = (struct clocksource *) data;
1362
1363 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1364 write_seqcount_begin(&tk_core.seq);
1365
1366 timekeeping_forward_now(tk);
1367
1368
1369
1370
1371 if (try_module_get(new->owner)) {
1372 if (!new->enable || new->enable(new) == 0) {
1373 old = tk->tkr_mono.clock;
1374 tk_setup_internals(tk, new);
1375 if (old->disable)
1376 old->disable(old);
1377 module_put(old->owner);
1378 } else {
1379 module_put(new->owner);
1380 }
1381 }
1382 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1383
1384 write_seqcount_end(&tk_core.seq);
1385 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1386
1387 return 0;
1388}
1389
1390
1391
1392
1393
1394
1395
1396
1397int timekeeping_notify(struct clocksource *clock)
1398{
1399 struct timekeeper *tk = &tk_core.timekeeper;
1400
1401 if (tk->tkr_mono.clock == clock)
1402 return 0;
1403 stop_machine(change_clocksource, clock, NULL);
1404 tick_clock_notify();
1405 return tk->tkr_mono.clock == clock ? 0 : -1;
1406}
1407
1408
1409
1410
1411
1412
1413
1414void ktime_get_raw_ts64(struct timespec64 *ts)
1415{
1416 struct timekeeper *tk = &tk_core.timekeeper;
1417 unsigned int seq;
1418 u64 nsecs;
1419
1420 do {
1421 seq = read_seqcount_begin(&tk_core.seq);
1422 ts->tv_sec = tk->raw_sec;
1423 nsecs = timekeeping_get_ns(&tk->tkr_raw);
1424
1425 } while (read_seqcount_retry(&tk_core.seq, seq));
1426
1427 ts->tv_nsec = 0;
1428 timespec64_add_ns(ts, nsecs);
1429}
1430EXPORT_SYMBOL(ktime_get_raw_ts64);
1431
1432
1433
1434
1435
1436int timekeeping_valid_for_hres(void)
1437{
1438 struct timekeeper *tk = &tk_core.timekeeper;
1439 unsigned int seq;
1440 int ret;
1441
1442 do {
1443 seq = read_seqcount_begin(&tk_core.seq);
1444
1445 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1446
1447 } while (read_seqcount_retry(&tk_core.seq, seq));
1448
1449 return ret;
1450}
1451
1452
1453
1454
1455u64 timekeeping_max_deferment(void)
1456{
1457 struct timekeeper *tk = &tk_core.timekeeper;
1458 unsigned int seq;
1459 u64 ret;
1460
1461 do {
1462 seq = read_seqcount_begin(&tk_core.seq);
1463
1464 ret = tk->tkr_mono.clock->max_idle_ns;
1465
1466 } while (read_seqcount_retry(&tk_core.seq, seq));
1467
1468 return ret;
1469}
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480void __weak read_persistent_clock64(struct timespec64 *ts)
1481{
1482 ts->tv_sec = 0;
1483 ts->tv_nsec = 0;
1484}
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498void __weak __init
1499read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1500 struct timespec64 *boot_offset)
1501{
1502 read_persistent_clock64(wall_time);
1503 *boot_offset = ns_to_timespec64(local_clock());
1504}
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519static bool suspend_timing_needed;
1520
1521
1522static bool persistent_clock_exists;
1523
1524
1525
1526
1527void __init timekeeping_init(void)
1528{
1529 struct timespec64 wall_time, boot_offset, wall_to_mono;
1530 struct timekeeper *tk = &tk_core.timekeeper;
1531 struct clocksource *clock;
1532 unsigned long flags;
1533
1534 read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
1535 if (timespec64_valid_settod(&wall_time) &&
1536 timespec64_to_ns(&wall_time) > 0) {
1537 persistent_clock_exists = true;
1538 } else if (timespec64_to_ns(&wall_time) != 0) {
1539 pr_warn("Persistent clock returned invalid value");
1540 wall_time = (struct timespec64){0};
1541 }
1542
1543 if (timespec64_compare(&wall_time, &boot_offset) < 0)
1544 boot_offset = (struct timespec64){0};
1545
1546
1547
1548
1549
1550 wall_to_mono = timespec64_sub(boot_offset, wall_time);
1551
1552 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1553 write_seqcount_begin(&tk_core.seq);
1554 ntp_init();
1555
1556 clock = clocksource_default_clock();
1557 if (clock->enable)
1558 clock->enable(clock);
1559 tk_setup_internals(tk, clock);
1560
1561 tk_set_xtime(tk, &wall_time);
1562 tk->raw_sec = 0;
1563
1564 tk_set_wall_to_mono(tk, wall_to_mono);
1565
1566 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1567
1568 write_seqcount_end(&tk_core.seq);
1569 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1570}
1571
1572
1573static struct timespec64 timekeeping_suspend_time;
1574
1575
1576
1577
1578
1579
1580
1581
1582static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1583 const struct timespec64 *delta)
1584{
1585 if (!timespec64_valid_strict(delta)) {
1586 printk_deferred(KERN_WARNING
1587 "__timekeeping_inject_sleeptime: Invalid "
1588 "sleep delta value!\n");
1589 return;
1590 }
1591 tk_xtime_add(tk, delta);
1592 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1593 tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1594 tk_debug_account_sleep_time(delta);
1595}
1596
1597#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614bool timekeeping_rtc_skipresume(void)
1615{
1616 return !suspend_timing_needed;
1617}
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628bool timekeeping_rtc_skipsuspend(void)
1629{
1630 return persistent_clock_exists;
1631}
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
1645{
1646 struct timekeeper *tk = &tk_core.timekeeper;
1647 unsigned long flags;
1648
1649 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1650 write_seqcount_begin(&tk_core.seq);
1651
1652 suspend_timing_needed = false;
1653
1654 timekeeping_forward_now(tk);
1655
1656 __timekeeping_inject_sleeptime(tk, delta);
1657
1658 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1659
1660 write_seqcount_end(&tk_core.seq);
1661 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1662
1663
1664 clock_was_set();
1665}
1666#endif
1667
1668
1669
1670
1671void timekeeping_resume(void)
1672{
1673 struct timekeeper *tk = &tk_core.timekeeper;
1674 struct clocksource *clock = tk->tkr_mono.clock;
1675 unsigned long flags;
1676 struct timespec64 ts_new, ts_delta;
1677 u64 cycle_now, nsec;
1678 bool inject_sleeptime = false;
1679
1680 read_persistent_clock64(&ts_new);
1681
1682 clockevents_resume();
1683 clocksource_resume();
1684
1685 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1686 write_seqcount_begin(&tk_core.seq);
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700 cycle_now = tk_clock_read(&tk->tkr_mono);
1701 nsec = clocksource_stop_suspend_timing(clock, cycle_now);
1702 if (nsec > 0) {
1703 ts_delta = ns_to_timespec64(nsec);
1704 inject_sleeptime = true;
1705 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1706 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1707 inject_sleeptime = true;
1708 }
1709
1710 if (inject_sleeptime) {
1711 suspend_timing_needed = false;
1712 __timekeeping_inject_sleeptime(tk, &ts_delta);
1713 }
1714
1715
1716 tk->tkr_mono.cycle_last = cycle_now;
1717 tk->tkr_raw.cycle_last = cycle_now;
1718
1719 tk->ntp_error = 0;
1720 timekeeping_suspended = 0;
1721 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1722 write_seqcount_end(&tk_core.seq);
1723 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1724
1725 touch_softlockup_watchdog();
1726
1727 tick_resume();
1728 hrtimers_resume();
1729}
1730
1731int timekeeping_suspend(void)
1732{
1733 struct timekeeper *tk = &tk_core.timekeeper;
1734 unsigned long flags;
1735 struct timespec64 delta, delta_delta;
1736 static struct timespec64 old_delta;
1737 struct clocksource *curr_clock;
1738 u64 cycle_now;
1739
1740 read_persistent_clock64(&timekeeping_suspend_time);
1741
1742
1743
1744
1745
1746
1747 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1748 persistent_clock_exists = true;
1749
1750 suspend_timing_needed = true;
1751
1752 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1753 write_seqcount_begin(&tk_core.seq);
1754 timekeeping_forward_now(tk);
1755 timekeeping_suspended = 1;
1756
1757
1758
1759
1760
1761
1762 curr_clock = tk->tkr_mono.clock;
1763 cycle_now = tk->tkr_mono.cycle_last;
1764 clocksource_start_suspend_timing(curr_clock, cycle_now);
1765
1766 if (persistent_clock_exists) {
1767
1768
1769
1770
1771
1772
1773 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1774 delta_delta = timespec64_sub(delta, old_delta);
1775 if (abs(delta_delta.tv_sec) >= 2) {
1776
1777
1778
1779
1780 old_delta = delta;
1781 } else {
1782
1783 timekeeping_suspend_time =
1784 timespec64_add(timekeeping_suspend_time, delta_delta);
1785 }
1786 }
1787
1788 timekeeping_update(tk, TK_MIRROR);
1789 halt_fast_timekeeper(tk);
1790 write_seqcount_end(&tk_core.seq);
1791 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1792
1793 tick_suspend();
1794 clocksource_suspend();
1795 clockevents_suspend();
1796
1797 return 0;
1798}
1799
1800
1801static struct syscore_ops timekeeping_syscore_ops = {
1802 .resume = timekeeping_resume,
1803 .suspend = timekeeping_suspend,
1804};
1805
1806static int __init timekeeping_init_ops(void)
1807{
1808 register_syscore_ops(&timekeeping_syscore_ops);
1809 return 0;
1810}
1811device_initcall(timekeeping_init_ops);
1812
1813
1814
1815
1816static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1817 s64 offset,
1818 s32 mult_adj)
1819{
1820 s64 interval = tk->cycle_interval;
1821
1822 if (mult_adj == 0) {
1823 return;
1824 } else if (mult_adj == -1) {
1825 interval = -interval;
1826 offset = -offset;
1827 } else if (mult_adj != 1) {
1828 interval *= mult_adj;
1829 offset *= mult_adj;
1830 }
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1880
1881 WARN_ON_ONCE(1);
1882 return;
1883 }
1884
1885 tk->tkr_mono.mult += mult_adj;
1886 tk->xtime_interval += interval;
1887 tk->tkr_mono.xtime_nsec -= offset;
1888}
1889
1890
1891
1892
1893
1894static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1895{
1896 u32 mult;
1897
1898
1899
1900
1901
1902 if (likely(tk->ntp_tick == ntp_tick_length())) {
1903 mult = tk->tkr_mono.mult - tk->ntp_err_mult;
1904 } else {
1905 tk->ntp_tick = ntp_tick_length();
1906 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
1907 tk->xtime_remainder, tk->cycle_interval);
1908 }
1909
1910
1911
1912
1913
1914
1915
1916 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
1917 mult += tk->ntp_err_mult;
1918
1919 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
1920
1921 if (unlikely(tk->tkr_mono.clock->maxadj &&
1922 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
1923 > tk->tkr_mono.clock->maxadj))) {
1924 printk_once(KERN_WARNING
1925 "Adjusting %s more than 11%% (%ld vs %ld)\n",
1926 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
1927 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
1928 }
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
1941 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
1942 tk->tkr_mono.shift;
1943 tk->xtime_sec--;
1944 tk->skip_second_overflow = 1;
1945 }
1946}
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1957{
1958 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
1959 unsigned int clock_set = 0;
1960
1961 while (tk->tkr_mono.xtime_nsec >= nsecps) {
1962 int leap;
1963
1964 tk->tkr_mono.xtime_nsec -= nsecps;
1965 tk->xtime_sec++;
1966
1967
1968
1969
1970
1971 if (unlikely(tk->skip_second_overflow)) {
1972 tk->skip_second_overflow = 0;
1973 continue;
1974 }
1975
1976
1977 leap = second_overflow(tk->xtime_sec);
1978 if (unlikely(leap)) {
1979 struct timespec64 ts;
1980
1981 tk->xtime_sec += leap;
1982
1983 ts.tv_sec = leap;
1984 ts.tv_nsec = 0;
1985 tk_set_wall_to_mono(tk,
1986 timespec64_sub(tk->wall_to_monotonic, ts));
1987
1988 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1989
1990 clock_set = TK_CLOCK_WAS_SET;
1991 }
1992 }
1993 return clock_set;
1994}
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2006 u32 shift, unsigned int *clock_set)
2007{
2008 u64 interval = tk->cycle_interval << shift;
2009 u64 snsec_per_sec;
2010
2011
2012 if (offset < interval)
2013 return offset;
2014
2015
2016 offset -= interval;
2017 tk->tkr_mono.cycle_last += interval;
2018 tk->tkr_raw.cycle_last += interval;
2019
2020 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2021 *clock_set |= accumulate_nsecs_to_secs(tk);
2022
2023
2024 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2025 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2026 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2027 tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2028 tk->raw_sec++;
2029 }
2030
2031
2032 tk->ntp_error += tk->ntp_tick << shift;
2033 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2034 (tk->ntp_error_shift + shift);
2035
2036 return offset;
2037}
2038
2039
2040
2041
2042
2043static void timekeeping_advance(enum timekeeping_adv_mode mode)
2044{
2045 struct timekeeper *real_tk = &tk_core.timekeeper;
2046 struct timekeeper *tk = &shadow_timekeeper;
2047 u64 offset;
2048 int shift = 0, maxshift;
2049 unsigned int clock_set = 0;
2050 unsigned long flags;
2051
2052 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2053
2054
2055 if (unlikely(timekeeping_suspended))
2056 goto out;
2057
2058#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
2059 offset = real_tk->cycle_interval;
2060
2061 if (mode != TK_ADV_TICK)
2062 goto out;
2063#else
2064 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
2065 tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
2066
2067
2068 if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
2069 goto out;
2070#endif
2071
2072
2073 timekeeping_check_update(tk, offset);
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083 shift = ilog2(offset) - ilog2(tk->cycle_interval);
2084 shift = max(0, shift);
2085
2086 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
2087 shift = min(shift, maxshift);
2088 while (offset >= tk->cycle_interval) {
2089 offset = logarithmic_accumulation(tk, offset, shift,
2090 &clock_set);
2091 if (offset < tk->cycle_interval<<shift)
2092 shift--;
2093 }
2094
2095
2096 timekeeping_adjust(tk, offset);
2097
2098
2099
2100
2101
2102 clock_set |= accumulate_nsecs_to_secs(tk);
2103
2104 write_seqcount_begin(&tk_core.seq);
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115 timekeeping_update(tk, clock_set);
2116 memcpy(real_tk, tk, sizeof(*tk));
2117
2118 write_seqcount_end(&tk_core.seq);
2119out:
2120 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2121 if (clock_set)
2122
2123 clock_was_set_delayed();
2124}
2125
2126
2127
2128
2129
2130void update_wall_time(void)
2131{
2132 timekeeping_advance(TK_ADV_TICK);
2133}
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146void getboottime64(struct timespec64 *ts)
2147{
2148 struct timekeeper *tk = &tk_core.timekeeper;
2149 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2150
2151 *ts = ktime_to_timespec64(t);
2152}
2153EXPORT_SYMBOL_GPL(getboottime64);
2154
2155void ktime_get_coarse_real_ts64(struct timespec64 *ts)
2156{
2157 struct timekeeper *tk = &tk_core.timekeeper;
2158 unsigned int seq;
2159
2160 do {
2161 seq = read_seqcount_begin(&tk_core.seq);
2162
2163 *ts = tk_xtime(tk);
2164 } while (read_seqcount_retry(&tk_core.seq, seq));
2165}
2166EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
2167
2168void ktime_get_coarse_ts64(struct timespec64 *ts)
2169{
2170 struct timekeeper *tk = &tk_core.timekeeper;
2171 struct timespec64 now, mono;
2172 unsigned int seq;
2173
2174 do {
2175 seq = read_seqcount_begin(&tk_core.seq);
2176
2177 now = tk_xtime(tk);
2178 mono = tk->wall_to_monotonic;
2179 } while (read_seqcount_retry(&tk_core.seq, seq));
2180
2181 set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
2182 now.tv_nsec + mono.tv_nsec);
2183}
2184EXPORT_SYMBOL(ktime_get_coarse_ts64);
2185
2186
2187
2188
2189void do_timer(unsigned long ticks)
2190{
2191 jiffies_64 += ticks;
2192 calc_global_load(ticks);
2193}
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2209 ktime_t *offs_boot, ktime_t *offs_tai)
2210{
2211 struct timekeeper *tk = &tk_core.timekeeper;
2212 unsigned int seq;
2213 ktime_t base;
2214 u64 nsecs;
2215
2216 do {
2217 seq = read_seqcount_begin(&tk_core.seq);
2218
2219 base = tk->tkr_mono.base;
2220 nsecs = timekeeping_get_ns(&tk->tkr_mono);
2221 base = ktime_add_ns(base, nsecs);
2222
2223 if (*cwsseq != tk->clock_was_set_seq) {
2224 *cwsseq = tk->clock_was_set_seq;
2225 *offs_real = tk->offs_real;
2226 *offs_boot = tk->offs_boot;
2227 *offs_tai = tk->offs_tai;
2228 }
2229
2230
2231 if (unlikely(base >= tk->next_leap_ktime))
2232 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2233
2234 } while (read_seqcount_retry(&tk_core.seq, seq));
2235
2236 return base;
2237}
2238
2239
2240
2241
2242static int timekeeping_validate_timex(const struct __kernel_timex *txc)
2243{
2244 if (txc->modes & ADJ_ADJTIME) {
2245
2246 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2247 return -EINVAL;
2248 if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2249 !capable(CAP_SYS_TIME))
2250 return -EPERM;
2251 } else {
2252
2253 if (txc->modes && !capable(CAP_SYS_TIME))
2254 return -EPERM;
2255
2256
2257
2258
2259 if (txc->modes & ADJ_TICK &&
2260 (txc->tick < 900000/USER_HZ ||
2261 txc->tick > 1100000/USER_HZ))
2262 return -EINVAL;
2263 }
2264
2265 if (txc->modes & ADJ_SETOFFSET) {
2266
2267 if (!capable(CAP_SYS_TIME))
2268 return -EPERM;
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278 if (txc->time.tv_usec < 0)
2279 return -EINVAL;
2280
2281 if (txc->modes & ADJ_NANO) {
2282 if (txc->time.tv_usec >= NSEC_PER_SEC)
2283 return -EINVAL;
2284 } else {
2285 if (txc->time.tv_usec >= USEC_PER_SEC)
2286 return -EINVAL;
2287 }
2288 }
2289
2290
2291
2292
2293
2294 if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
2295 if (LLONG_MIN / PPM_SCALE > txc->freq)
2296 return -EINVAL;
2297 if (LLONG_MAX / PPM_SCALE < txc->freq)
2298 return -EINVAL;
2299 }
2300
2301 return 0;
2302}
2303
2304
2305
2306
2307
2308int do_adjtimex(struct __kernel_timex *txc)
2309{
2310 struct timekeeper *tk = &tk_core.timekeeper;
2311 struct audit_ntp_data ad;
2312 unsigned long flags;
2313 struct timespec64 ts;
2314 s32 orig_tai, tai;
2315 int ret;
2316
2317
2318 ret = timekeeping_validate_timex(txc);
2319 if (ret)
2320 return ret;
2321
2322 if (txc->modes & ADJ_SETOFFSET) {
2323 struct timespec64 delta;
2324 delta.tv_sec = txc->time.tv_sec;
2325 delta.tv_nsec = txc->time.tv_usec;
2326 if (!(txc->modes & ADJ_NANO))
2327 delta.tv_nsec *= 1000;
2328 ret = timekeeping_inject_offset(&delta);
2329 if (ret)
2330 return ret;
2331
2332 audit_tk_injoffset(delta);
2333 }
2334
2335 audit_ntp_init(&ad);
2336
2337 ktime_get_real_ts64(&ts);
2338
2339 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2340 write_seqcount_begin(&tk_core.seq);
2341
2342 orig_tai = tai = tk->tai_offset;
2343 ret = __do_adjtimex(txc, &ts, &tai, &ad);
2344
2345 if (tai != orig_tai) {
2346 __timekeeping_set_tai_offset(tk, tai);
2347 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2348 }
2349 tk_update_leap_state(tk);
2350
2351 write_seqcount_end(&tk_core.seq);
2352 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2353
2354 audit_ntp_log(&ad);
2355
2356
2357 if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
2358 timekeeping_advance(TK_ADV_FREQ);
2359
2360 if (tai != orig_tai)
2361 clock_was_set();
2362
2363 ntp_notify_cmos_timer();
2364
2365 return ret;
2366}
2367
2368#ifdef CONFIG_NTP_PPS
2369
2370
2371
2372void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2373{
2374 unsigned long flags;
2375
2376 raw_spin_lock_irqsave(&timekeeper_lock, flags);
2377 write_seqcount_begin(&tk_core.seq);
2378
2379 __hardpps(phase_ts, raw_ts);
2380
2381 write_seqcount_end(&tk_core.seq);
2382 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2383}
2384EXPORT_SYMBOL(hardpps);
2385#endif
2386
2387
2388
2389
2390
2391
2392
2393void xtime_update(unsigned long ticks)
2394{
2395 write_seqlock(&jiffies_lock);
2396 do_timer(ticks);
2397 write_sequnlock(&jiffies_lock);
2398 update_wall_time();
2399}
2400