1#include <linux/kernel.h>
2#include <linux/sched.h>
3#include <linux/init.h>
4#include <linux/module.h>
5#include <linux/timer.h>
6#include <linux/acpi_pmtmr.h>
7#include <linux/cpufreq.h>
8#include <linux/dmi.h>
9#include <linux/delay.h>
10#include <linux/clocksource.h>
11#include <linux/percpu.h>
12#include <linux/timex.h>
13
14#include <asm/hpet.h>
15#include <asm/timer.h>
16#include <asm/vgtod.h>
17#include <asm/time.h>
18#include <asm/delay.h>
19#include <asm/hypervisor.h>
20#include <asm/nmi.h>
21#include <asm/x86_init.h>
22
23unsigned int __read_mostly cpu_khz;
24EXPORT_SYMBOL(cpu_khz);
25
26unsigned int __read_mostly tsc_khz;
27EXPORT_SYMBOL(tsc_khz);
28
29
30
31
32static int __read_mostly tsc_unstable;
33
34
35
36
37static int __read_mostly tsc_disabled = -1;
38
39static int tsc_clocksource_reliable;
40
41
42
43u64 native_sched_clock(void)
44{
45 u64 this_offset;
46
47
48
49
50
51
52
53
54
55 if (unlikely(tsc_disabled)) {
56
57 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
58 }
59
60
61 rdtscll(this_offset);
62
63
64 return __cycles_2_ns(this_offset);
65}
66
67
68
69#ifdef CONFIG_PARAVIRT
70unsigned long long sched_clock(void)
71{
72 return paravirt_sched_clock();
73}
74#else
75unsigned long long
76sched_clock(void) __attribute__((alias("native_sched_clock")));
77#endif
78
79int check_tsc_unstable(void)
80{
81 return tsc_unstable;
82}
83EXPORT_SYMBOL_GPL(check_tsc_unstable);
84
85#ifdef CONFIG_X86_TSC
86int __init notsc_setup(char *str)
87{
88 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
89 "cannot disable TSC completely.\n");
90 tsc_disabled = 1;
91 return 1;
92}
93#else
94
95
96
97
98int __init notsc_setup(char *str)
99{
100 setup_clear_cpu_cap(X86_FEATURE_TSC);
101 return 1;
102}
103#endif
104
105__setup("notsc", notsc_setup);
106
107static int __init tsc_setup(char *str)
108{
109 if (!strcmp(str, "reliable"))
110 tsc_clocksource_reliable = 1;
111 return 1;
112}
113
114__setup("tsc=", tsc_setup);
115
116#define MAX_RETRIES 5
117#define SMI_TRESHOLD 50000
118
119
120
121
122static u64 tsc_read_refs(u64 *p, int hpet)
123{
124 u64 t1, t2;
125 int i;
126
127 for (i = 0; i < MAX_RETRIES; i++) {
128 t1 = get_cycles();
129 if (hpet)
130 *p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
131 else
132 *p = acpi_pm_read_early();
133 t2 = get_cycles();
134 if ((t2 - t1) < SMI_TRESHOLD)
135 return t2;
136 }
137 return ULLONG_MAX;
138}
139
140
141
142
143static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2)
144{
145 u64 tmp;
146
147 if (hpet2 < hpet1)
148 hpet2 += 0x100000000ULL;
149 hpet2 -= hpet1;
150 tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
151 do_div(tmp, 1000000);
152 do_div(deltatsc, tmp);
153
154 return (unsigned long) deltatsc;
155}
156
157
158
159
160static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2)
161{
162 u64 tmp;
163
164 if (!pm1 && !pm2)
165 return ULONG_MAX;
166
167 if (pm2 < pm1)
168 pm2 += (u64)ACPI_PM_OVRRUN;
169 pm2 -= pm1;
170 tmp = pm2 * 1000000000LL;
171 do_div(tmp, PMTMR_TICKS_PER_SEC);
172 do_div(deltatsc, tmp);
173
174 return (unsigned long) deltatsc;
175}
176
177#define CAL_MS 10
178#define CAL_LATCH (CLOCK_TICK_RATE / (1000 / CAL_MS))
179#define CAL_PIT_LOOPS 1000
180
181#define CAL2_MS 50
182#define CAL2_LATCH (CLOCK_TICK_RATE / (1000 / CAL2_MS))
183#define CAL2_PIT_LOOPS 5000
184
185
186
187
188
189
190
191
192
193static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
194{
195 u64 tsc, t1, t2, delta;
196 unsigned long tscmin, tscmax;
197 int pitcnt;
198
199
200 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
201
202
203
204
205
206
207 outb(0xb0, 0x43);
208 outb(latch & 0xff, 0x42);
209 outb(latch >> 8, 0x42);
210
211 tsc = t1 = t2 = get_cycles();
212
213 pitcnt = 0;
214 tscmax = 0;
215 tscmin = ULONG_MAX;
216 while ((inb(0x61) & 0x20) == 0) {
217 t2 = get_cycles();
218 delta = t2 - tsc;
219 tsc = t2;
220 if ((unsigned long) delta < tscmin)
221 tscmin = (unsigned int) delta;
222 if ((unsigned long) delta > tscmax)
223 tscmax = (unsigned int) delta;
224 pitcnt++;
225 }
226
227
228
229
230
231
232
233
234
235
236 if (pitcnt < loopmin || tscmax > 10 * tscmin)
237 return ULONG_MAX;
238
239
240 delta = t2 - t1;
241 do_div(delta, ms);
242 return delta;
243}
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280static inline int pit_verify_msb(unsigned char val)
281{
282
283 inb(0x42);
284 return inb(0x42) == val;
285}
286
287static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *deltap)
288{
289 int count;
290 u64 tsc = 0;
291
292 for (count = 0; count < 50000; count++) {
293 if (!pit_verify_msb(val))
294 break;
295 tsc = get_cycles();
296 }
297 *deltap = get_cycles() - tsc;
298 *tscp = tsc;
299
300
301
302
303
304 return count > 5;
305}
306
307
308
309
310
311
312
313#define MAX_QUICK_PIT_MS 25
314#define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
315
316static unsigned long quick_pit_calibrate(void)
317{
318 int i;
319 u64 tsc, delta;
320 unsigned long d1, d2;
321
322
323 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
324
325
326
327
328
329
330
331
332
333
334 outb(0xb0, 0x43);
335
336
337 outb(0xff, 0x42);
338 outb(0xff, 0x42);
339
340
341
342
343
344
345
346 pit_verify_msb(0);
347
348 if (pit_expect_msb(0xff, &tsc, &d1)) {
349 for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) {
350 if (!pit_expect_msb(0xff-i, &delta, &d2))
351 break;
352
353
354
355
356 delta -= tsc;
357 if (d1+d2 >= delta >> 11)
358 continue;
359
360
361
362
363
364
365
366
367 if (!pit_verify_msb(0xfe - i))
368 break;
369 goto success;
370 }
371 }
372 printk("Fast TSC calibration failed\n");
373 return 0;
374
375success:
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391 delta += (long)(d2 - d1)/2;
392 delta *= PIT_TICK_RATE;
393 do_div(delta, i*256*1000);
394 printk("Fast TSC calibration using PIT\n");
395 return delta;
396}
397
398
399
400
401unsigned long native_calibrate_tsc(void)
402{
403 u64 tsc1, tsc2, delta, ref1, ref2;
404 unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
405 unsigned long flags, latch, ms, fast_calibrate;
406 int hpet = is_hpet_enabled(), i, loopmin;
407
408 local_irq_save(flags);
409 fast_calibrate = quick_pit_calibrate();
410 local_irq_restore(flags);
411 if (fast_calibrate)
412 return fast_calibrate;
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440 latch = CAL_LATCH;
441 ms = CAL_MS;
442 loopmin = CAL_PIT_LOOPS;
443
444 for (i = 0; i < 3; i++) {
445 unsigned long tsc_pit_khz;
446
447
448
449
450
451
452
453 local_irq_save(flags);
454 tsc1 = tsc_read_refs(&ref1, hpet);
455 tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
456 tsc2 = tsc_read_refs(&ref2, hpet);
457 local_irq_restore(flags);
458
459
460 tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
461
462
463 if (!hpet && !ref1 && !ref2)
464 continue;
465
466
467 if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX)
468 continue;
469
470 tsc2 = (tsc2 - tsc1) * 1000000LL;
471 if (hpet)
472 tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
473 else
474 tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
475
476 tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
477
478
479 delta = ((u64) tsc_pit_min) * 100;
480 do_div(delta, tsc_ref_min);
481
482
483
484
485
486
487
488 if (delta >= 90 && delta <= 110) {
489 printk(KERN_INFO
490 "TSC: PIT calibration matches %s. %d loops\n",
491 hpet ? "HPET" : "PMTIMER", i + 1);
492 return tsc_ref_min;
493 }
494
495
496
497
498
499
500
501 if (i == 1 && tsc_pit_min == ULONG_MAX) {
502 latch = CAL2_LATCH;
503 ms = CAL2_MS;
504 loopmin = CAL2_PIT_LOOPS;
505 }
506 }
507
508
509
510
511 if (tsc_pit_min == ULONG_MAX) {
512
513 printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n");
514
515
516 if (!hpet && !ref1 && !ref2) {
517 printk("TSC: No reference (HPET/PMTIMER) available\n");
518 return 0;
519 }
520
521
522 if (tsc_ref_min == ULONG_MAX) {
523 printk(KERN_WARNING "TSC: HPET/PMTIMER calibration "
524 "failed.\n");
525 return 0;
526 }
527
528
529 printk(KERN_INFO "TSC: using %s reference calibration\n",
530 hpet ? "HPET" : "PMTIMER");
531
532 return tsc_ref_min;
533 }
534
535
536 if (!hpet && !ref1 && !ref2) {
537 printk(KERN_INFO "TSC: Using PIT calibration value\n");
538 return tsc_pit_min;
539 }
540
541
542 if (tsc_ref_min == ULONG_MAX) {
543 printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed. "
544 "Using PIT calibration\n");
545 return tsc_pit_min;
546 }
547
548
549
550
551
552
553 printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n",
554 hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
555 printk(KERN_INFO "TSC: Using PIT calibration value\n");
556 return tsc_pit_min;
557}
558
559int recalibrate_cpu_khz(void)
560{
561#ifndef CONFIG_SMP
562 unsigned long cpu_khz_old = cpu_khz;
563
564 if (cpu_has_tsc) {
565 tsc_khz = x86_platform.calibrate_tsc();
566 cpu_khz = tsc_khz;
567 cpu_data(0).loops_per_jiffy =
568 cpufreq_scale(cpu_data(0).loops_per_jiffy,
569 cpu_khz_old, cpu_khz);
570 return 0;
571 } else
572 return -ENODEV;
573#else
574 return -ENODEV;
575#endif
576}
577
578EXPORT_SYMBOL(recalibrate_cpu_khz);
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603DEFINE_PER_CPU(unsigned long, cyc2ns);
604DEFINE_PER_CPU(unsigned long long, cyc2ns_offset);
605
606static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
607{
608 unsigned long long tsc_now, ns_now, *offset;
609 unsigned long flags, *scale;
610
611 local_irq_save(flags);
612 sched_clock_idle_sleep_event();
613
614 scale = &per_cpu(cyc2ns, cpu);
615 offset = &per_cpu(cyc2ns_offset, cpu);
616
617 rdtscll(tsc_now);
618 ns_now = __cycles_2_ns(tsc_now);
619
620 if (cpu_khz) {
621 *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
622 *offset = ns_now - (tsc_now * *scale >> CYC2NS_SCALE_FACTOR);
623 }
624
625 sched_clock_idle_wakeup_event(0);
626 local_irq_restore(flags);
627}
628
629#ifdef CONFIG_CPU_FREQ
630
631
632
633
634
635
636
637
638
639
640
641
642static unsigned int ref_freq;
643static unsigned long loops_per_jiffy_ref;
644static unsigned long tsc_khz_ref;
645
646static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
647 void *data)
648{
649 struct cpufreq_freqs *freq = data;
650 unsigned long *lpj;
651
652 if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC))
653 return 0;
654
655 lpj = &boot_cpu_data.loops_per_jiffy;
656#ifdef CONFIG_SMP
657 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
658 lpj = &cpu_data(freq->cpu).loops_per_jiffy;
659#endif
660
661 if (!ref_freq) {
662 ref_freq = freq->old;
663 loops_per_jiffy_ref = *lpj;
664 tsc_khz_ref = tsc_khz;
665 }
666 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
667 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
668 (val == CPUFREQ_RESUMECHANGE)) {
669 *lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
670
671 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
672 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
673 mark_tsc_unstable("cpufreq changes");
674 }
675
676 set_cyc2ns_scale(tsc_khz, freq->cpu);
677
678 return 0;
679}
680
681static struct notifier_block time_cpufreq_notifier_block = {
682 .notifier_call = time_cpufreq_notifier
683};
684
685static int __init cpufreq_tsc(void)
686{
687 if (!cpu_has_tsc)
688 return 0;
689 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
690 return 0;
691 cpufreq_register_notifier(&time_cpufreq_notifier_block,
692 CPUFREQ_TRANSITION_NOTIFIER);
693 return 0;
694}
695
696core_initcall(cpufreq_tsc);
697
698#endif
699
700
701
702static struct clocksource clocksource_tsc;
703
704
705
706
707
708
709
710
711
712
713
714
715
716static cycle_t read_tsc(struct clocksource *cs)
717{
718 cycle_t ret = (cycle_t)get_cycles();
719
720 return ret >= clocksource_tsc.cycle_last ?
721 ret : clocksource_tsc.cycle_last;
722}
723
724#ifdef CONFIG_X86_64
725static cycle_t __vsyscall_fn vread_tsc(void)
726{
727 cycle_t ret;
728
729
730
731
732
733
734 rdtsc_barrier();
735 ret = (cycle_t)vget_cycles();
736 rdtsc_barrier();
737
738 return ret >= __vsyscall_gtod_data.clock.cycle_last ?
739 ret : __vsyscall_gtod_data.clock.cycle_last;
740}
741#endif
742
743static void resume_tsc(void)
744{
745 clocksource_tsc.cycle_last = 0;
746}
747
748static struct clocksource clocksource_tsc = {
749 .name = "tsc",
750 .rating = 300,
751 .read = read_tsc,
752 .resume = resume_tsc,
753 .mask = CLOCKSOURCE_MASK(64),
754 .shift = 22,
755 .flags = CLOCK_SOURCE_IS_CONTINUOUS |
756 CLOCK_SOURCE_MUST_VERIFY,
757#ifdef CONFIG_X86_64
758 .vread = vread_tsc,
759#endif
760};
761
762void mark_tsc_unstable(char *reason)
763{
764 if (!tsc_unstable) {
765 tsc_unstable = 1;
766 printk(KERN_INFO "Marking TSC unstable due to %s\n", reason);
767
768 if (clocksource_tsc.mult)
769 clocksource_mark_unstable(&clocksource_tsc);
770 else {
771 clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE;
772 clocksource_tsc.rating = 0;
773 }
774 }
775}
776
777EXPORT_SYMBOL_GPL(mark_tsc_unstable);
778
779static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
780{
781 printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
782 d->ident);
783 tsc_unstable = 1;
784 return 0;
785}
786
787
788static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
789 {
790 .callback = dmi_mark_tsc_unstable,
791 .ident = "IBM Thinkpad 380XD",
792 .matches = {
793 DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
794 DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
795 },
796 },
797 {}
798};
799
800static void __init check_system_tsc_reliable(void)
801{
802#ifdef CONFIG_MGEODE_LX
803
804#define RTSC_SUSP 0x100
805 unsigned long res_low, res_high;
806
807 rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
808
809 if (res_low & RTSC_SUSP)
810 tsc_clocksource_reliable = 1;
811#endif
812 if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
813 tsc_clocksource_reliable = 1;
814}
815
816
817
818
819
820__cpuinit int unsynchronized_tsc(void)
821{
822 if (!cpu_has_tsc || tsc_unstable)
823 return 1;
824
825#ifdef CONFIG_SMP
826 if (apic_is_clustered_box())
827 return 1;
828#endif
829
830 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
831 return 0;
832
833
834
835
836 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
837
838 if (num_possible_cpus() > 1)
839 tsc_unstable = 1;
840 }
841
842 return tsc_unstable;
843}
844
845static void __init init_tsc_clocksource(void)
846{
847 clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
848 clocksource_tsc.shift);
849 if (tsc_clocksource_reliable)
850 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
851
852 if (check_tsc_unstable()) {
853 clocksource_tsc.rating = 0;
854 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
855 }
856 clocksource_register(&clocksource_tsc);
857}
858
859#ifdef CONFIG_X86_64
860
861
862
863
864#define TICK_COUNT 100000000
865static unsigned long __init calibrate_cpu(void)
866{
867 int tsc_start, tsc_now;
868 int i, no_ctr_free;
869 unsigned long evntsel3 = 0, pmc3 = 0, pmc_now = 0;
870 unsigned long flags;
871
872 for (i = 0; i < 4; i++)
873 if (avail_to_resrv_perfctr_nmi_bit(i))
874 break;
875 no_ctr_free = (i == 4);
876 if (no_ctr_free) {
877 WARN(1, KERN_WARNING "Warning: AMD perfctrs busy ... "
878 "cpu_khz value may be incorrect.\n");
879 i = 3;
880 rdmsrl(MSR_K7_EVNTSEL3, evntsel3);
881 wrmsrl(MSR_K7_EVNTSEL3, 0);
882 rdmsrl(MSR_K7_PERFCTR3, pmc3);
883 } else {
884 reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i);
885 reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
886 }
887 local_irq_save(flags);
888
889 wrmsrl(MSR_K7_PERFCTR0 + i, 0);
890 wrmsrl(MSR_K7_EVNTSEL0 + i, 1 << 22 | 3 << 16 | 0x76);
891 rdtscl(tsc_start);
892 do {
893 rdmsrl(MSR_K7_PERFCTR0 + i, pmc_now);
894 tsc_now = get_cycles();
895 } while ((tsc_now - tsc_start) < TICK_COUNT);
896
897 local_irq_restore(flags);
898 if (no_ctr_free) {
899 wrmsrl(MSR_K7_EVNTSEL3, 0);
900 wrmsrl(MSR_K7_PERFCTR3, pmc3);
901 wrmsrl(MSR_K7_EVNTSEL3, evntsel3);
902 } else {
903 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
904 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
905 }
906
907 return pmc_now * tsc_khz / (tsc_now - tsc_start);
908}
909#else
910static inline unsigned long calibrate_cpu(void) { return cpu_khz; }
911#endif
912
913void __init tsc_init(void)
914{
915 u64 lpj;
916 int cpu;
917
918 x86_init.timers.tsc_pre_init();
919
920 if (!cpu_has_tsc)
921 return;
922
923 tsc_khz = x86_platform.calibrate_tsc();
924 cpu_khz = tsc_khz;
925
926 if (!tsc_khz) {
927 mark_tsc_unstable("could not calculate TSC khz");
928 return;
929 }
930
931 if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) &&
932 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD))
933 cpu_khz = calibrate_cpu();
934
935 printk("Detected %lu.%03lu MHz processor.\n",
936 (unsigned long)cpu_khz / 1000,
937 (unsigned long)cpu_khz % 1000);
938
939
940
941
942
943
944
945 for_each_possible_cpu(cpu)
946 set_cyc2ns_scale(cpu_khz, cpu);
947
948 if (tsc_disabled > 0)
949 return;
950
951
952 tsc_disabled = 0;
953
954 lpj = ((u64)tsc_khz * 1000);
955 do_div(lpj, HZ);
956 lpj_fine = lpj;
957
958 use_tsc_delay();
959
960 dmi_check_system(bad_tsc_dmi_table);
961
962 if (unsynchronized_tsc())
963 mark_tsc_unstable("TSCs unsynchronized");
964
965 check_system_tsc_reliable();
966 init_tsc_clocksource();
967}
968
969