1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/spinlock.h>
18#include <linux/kernel.h>
19#include <linux/smp.h>
20#include <linux/nmi.h>
21#include <asm/tsc.h>
22
23
24
25
26
27static atomic_t start_count;
28static atomic_t stop_count;
29
30
31
32
33
34
35static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
36
37static cycles_t last_tsc;
38static cycles_t max_warp;
39static int nr_warps;
40
41
42
43
44
45static void check_tsc_warp(unsigned int timeout)
46{
47 cycles_t start, now, prev, end;
48 int i;
49
50 start = rdtsc_ordered();
51
52
53
54 end = start + (cycles_t) tsc_khz * timeout;
55 now = start;
56
57 for (i = 0; ; i++) {
58
59
60
61
62
63 arch_spin_lock(&sync_lock);
64 prev = last_tsc;
65 now = rdtsc_ordered();
66 last_tsc = now;
67 arch_spin_unlock(&sync_lock);
68
69
70
71
72
73
74
75 if (unlikely(!(i & 7))) {
76 if (now > end || i > 10000000)
77 break;
78 cpu_relax();
79 touch_nmi_watchdog();
80 }
81
82
83
84
85 if (unlikely(prev > now)) {
86 arch_spin_lock(&sync_lock);
87 max_warp = max(max_warp, prev - now);
88 nr_warps++;
89 arch_spin_unlock(&sync_lock);
90 }
91 }
92 WARN(!(now-start),
93 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
94 now-start, end-start);
95}
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111static inline unsigned int loop_timeout(int cpu)
112{
113 return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20;
114}
115
116
117
118
119
120void check_tsc_sync_source(int cpu)
121{
122 int cpus = 2;
123
124
125
126
127
128 if (unsynchronized_tsc())
129 return;
130
131 if (tsc_clocksource_reliable) {
132 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
133 pr_info(
134 "Skipped synchronization checks as TSC is reliable.\n");
135 return;
136 }
137
138
139
140
141 atomic_set(&stop_count, 0);
142
143
144
145
146 while (atomic_read(&start_count) != cpus-1)
147 cpu_relax();
148
149
150
151 atomic_inc(&start_count);
152
153 check_tsc_warp(loop_timeout(cpu));
154
155 while (atomic_read(&stop_count) != cpus-1)
156 cpu_relax();
157
158 if (nr_warps) {
159 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
160 smp_processor_id(), cpu);
161 pr_warning("Measured %Ld cycles TSC warp between CPUs, "
162 "turning off TSC clock.\n", max_warp);
163 mark_tsc_unstable("check_tsc_sync_source failed");
164 } else {
165 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
166 smp_processor_id(), cpu);
167 }
168
169
170
171
172 atomic_set(&start_count, 0);
173 nr_warps = 0;
174 max_warp = 0;
175 last_tsc = 0;
176
177
178
179
180 atomic_inc(&stop_count);
181}
182
183
184
185
186void check_tsc_sync_target(void)
187{
188 int cpus = 2;
189
190
191 if (unsynchronized_tsc() || tsc_clocksource_reliable)
192 return;
193
194
195
196
197
198 atomic_inc(&start_count);
199 while (atomic_read(&start_count) != cpus)
200 cpu_relax();
201
202 check_tsc_warp(loop_timeout(smp_processor_id()));
203
204
205
206
207 atomic_inc(&stop_count);
208
209
210
211
212 while (atomic_read(&stop_count) != cpus)
213 cpu_relax();
214}
215