1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/spinlock.h>
18#include <linux/kernel.h>
19#include <linux/init.h>
20#include <linux/smp.h>
21#include <linux/nmi.h>
22#include <asm/tsc.h>
23
24
25
26
27
28static atomic_t start_count;
29static atomic_t stop_count;
30
31
32
33
34
35
36static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
37
38static cycles_t last_tsc;
39static cycles_t max_warp;
40static int nr_warps;
41
42
43
44
45static void check_tsc_warp(unsigned int timeout)
46{
47 cycles_t start, now, prev, end;
48 int i;
49
50 rdtsc_barrier();
51 start = get_cycles();
52 rdtsc_barrier();
53
54
55
56 end = start + (cycles_t) tsc_khz * timeout;
57 now = start;
58
59 for (i = 0; ; i++) {
60
61
62
63
64
65 arch_spin_lock(&sync_lock);
66 prev = last_tsc;
67 rdtsc_barrier();
68 now = get_cycles();
69 rdtsc_barrier();
70 last_tsc = now;
71 arch_spin_unlock(&sync_lock);
72
73
74
75
76
77
78
79 if (unlikely(!(i & 7))) {
80 if (now > end || i > 10000000)
81 break;
82 cpu_relax();
83 touch_nmi_watchdog();
84 }
85
86
87
88
89 if (unlikely(prev > now)) {
90 arch_spin_lock(&sync_lock);
91 max_warp = max(max_warp, prev - now);
92 nr_warps++;
93 arch_spin_unlock(&sync_lock);
94 }
95 }
96 WARN(!(now-start),
97 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
98 now-start, end-start);
99}
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115static inline unsigned int loop_timeout(int cpu)
116{
117 return (cpumask_weight(cpu_core_mask(cpu)) > 1) ? 2 : 20;
118}
119
120
121
122
123
124void check_tsc_sync_source(int cpu)
125{
126 int cpus = 2;
127
128
129
130
131
132 if (unsynchronized_tsc())
133 return;
134
135 if (tsc_clocksource_reliable) {
136 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
137 pr_info(
138 "Skipped synchronization checks as TSC is reliable.\n");
139 return;
140 }
141
142
143
144
145 atomic_set(&stop_count, 0);
146
147
148
149
150 while (atomic_read(&start_count) != cpus-1)
151 cpu_relax();
152
153
154
155 atomic_inc(&start_count);
156
157 check_tsc_warp(loop_timeout(cpu));
158
159 while (atomic_read(&stop_count) != cpus-1)
160 cpu_relax();
161
162 if (nr_warps) {
163 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
164 smp_processor_id(), cpu);
165 pr_warning("Measured %Ld cycles TSC warp between CPUs, "
166 "turning off TSC clock.\n", max_warp);
167 mark_tsc_unstable("check_tsc_sync_source failed");
168 } else {
169 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
170 smp_processor_id(), cpu);
171 }
172
173
174
175
176 atomic_set(&start_count, 0);
177 nr_warps = 0;
178 max_warp = 0;
179 last_tsc = 0;
180
181
182
183
184 atomic_inc(&stop_count);
185}
186
187
188
189
190void check_tsc_sync_target(void)
191{
192 int cpus = 2;
193
194 if (unsynchronized_tsc() || tsc_clocksource_reliable)
195 return;
196
197
198
199
200
201 atomic_inc(&start_count);
202 while (atomic_read(&start_count) != cpus)
203 cpu_relax();
204
205 check_tsc_warp(loop_timeout(smp_processor_id()));
206
207
208
209
210 atomic_inc(&stop_count);
211
212
213
214
215 while (atomic_read(&stop_count) != cpus)
216 cpu_relax();
217}
218