1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/topology.h>
19#include <linux/spinlock.h>
20#include <linux/kernel.h>
21#include <linux/smp.h>
22#include <linux/nmi.h>
23#include <asm/tsc.h>
24
25struct tsc_adjust {
26 s64 bootval;
27 s64 adjusted;
28 unsigned long nextcheck;
29 bool warned;
30};
31
32static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust);
33
34
35
36
37
38bool __read_mostly tsc_async_resets;
39
40void mark_tsc_async_resets(char *reason)
41{
42 if (tsc_async_resets)
43 return;
44 tsc_async_resets = true;
45 pr_info("tsc: Marking TSC async resets true due to %s\n", reason);
46}
47
48void tsc_verify_tsc_adjust(bool resume)
49{
50 struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust);
51 s64 curval;
52
53 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
54 return;
55
56
57 if (check_tsc_unstable())
58 return;
59
60
61 if (!resume && time_before(jiffies, adj->nextcheck))
62 return;
63
64 adj->nextcheck = jiffies + HZ;
65
66 rdmsrl(MSR_IA32_TSC_ADJUST, curval);
67 if (adj->adjusted == curval)
68 return;
69
70
71 wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted);
72
73 if (!adj->warned || resume) {
74 pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n",
75 smp_processor_id(), adj->adjusted, curval);
76 adj->warned = true;
77 }
78}
79
80static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval,
81 unsigned int cpu, bool bootcpu)
82{
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99 if (bootcpu && bootval != 0) {
100 if (likely(!tsc_async_resets)) {
101 pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n",
102 cpu, bootval);
103 wrmsrl(MSR_IA32_TSC_ADJUST, 0);
104 bootval = 0;
105 } else {
106 pr_info("TSC ADJUST: CPU%u: %lld NOT forced to 0\n",
107 cpu, bootval);
108 }
109 }
110 cur->adjusted = bootval;
111}
112
113#ifndef CONFIG_SMP
114bool __init tsc_store_and_check_tsc_adjust(bool bootcpu)
115{
116 struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust);
117 s64 bootval;
118
119 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
120 return false;
121
122
123 if (check_tsc_unstable())
124 return false;
125
126 rdmsrl(MSR_IA32_TSC_ADJUST, bootval);
127 cur->bootval = bootval;
128 cur->nextcheck = jiffies + HZ;
129 tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu);
130 return false;
131}
132
133#else
134
135
136
137
138bool tsc_store_and_check_tsc_adjust(bool bootcpu)
139{
140 struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust);
141 unsigned int refcpu, cpu = smp_processor_id();
142 struct cpumask *mask;
143 s64 bootval;
144
145 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
146 return false;
147
148 rdmsrl(MSR_IA32_TSC_ADJUST, bootval);
149 cur->bootval = bootval;
150 cur->nextcheck = jiffies + HZ;
151 cur->warned = false;
152
153
154
155
156
157 if (tsc_async_resets)
158 cur->adjusted = bootval;
159
160
161
162
163
164
165
166
167 mask = topology_core_cpumask(cpu);
168 refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids;
169
170 if (refcpu >= nr_cpu_ids) {
171 tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(),
172 bootcpu);
173 return false;
174 }
175
176 ref = per_cpu_ptr(&tsc_adjust, refcpu);
177
178
179
180
181 if (bootval != ref->bootval)
182 printk_once(FW_BUG "TSC ADJUST differs within socket(s), fixing all errors\n");
183
184
185
186
187
188
189
190 if (bootval != ref->adjusted) {
191 cur->adjusted = ref->adjusted;
192 wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted);
193 }
194
195
196
197
198 return true;
199}
200
201
202
203
204
205static atomic_t start_count;
206static atomic_t stop_count;
207static atomic_t skip_test;
208static atomic_t test_runs;
209
210
211
212
213
214
215static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
216
217static cycles_t last_tsc;
218static cycles_t max_warp;
219static int nr_warps;
220static int random_warps;
221
222
223
224
225
226static cycles_t check_tsc_warp(unsigned int timeout)
227{
228 cycles_t start, now, prev, end, cur_max_warp = 0;
229 int i, cur_warps = 0;
230
231 start = rdtsc_ordered();
232
233
234
235 end = start + (cycles_t) tsc_khz * timeout;
236
237 for (i = 0; ; i++) {
238
239
240
241
242
243 arch_spin_lock(&sync_lock);
244 prev = last_tsc;
245 now = rdtsc_ordered();
246 last_tsc = now;
247 arch_spin_unlock(&sync_lock);
248
249
250
251
252
253
254
255 if (unlikely(!(i & 7))) {
256 if (now > end || i > 10000000)
257 break;
258 cpu_relax();
259 touch_nmi_watchdog();
260 }
261
262
263
264
265 if (unlikely(prev > now)) {
266 arch_spin_lock(&sync_lock);
267 max_warp = max(max_warp, prev - now);
268 cur_max_warp = max_warp;
269
270
271
272
273 if (cur_warps != nr_warps)
274 random_warps++;
275 nr_warps++;
276 cur_warps = nr_warps;
277 arch_spin_unlock(&sync_lock);
278 }
279 }
280 WARN(!(now-start),
281 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
282 now-start, end-start);
283 return cur_max_warp;
284}
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300static inline unsigned int loop_timeout(int cpu)
301{
302 return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20;
303}
304
305
306
307
308
309void check_tsc_sync_source(int cpu)
310{
311 int cpus = 2;
312
313
314
315
316
317 if (unsynchronized_tsc())
318 return;
319
320
321
322
323
324
325 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
326 atomic_set(&test_runs, 1);
327 else
328 atomic_set(&test_runs, 3);
329retry:
330
331
332
333 while (atomic_read(&start_count) != cpus - 1) {
334 if (atomic_read(&skip_test) > 0) {
335 atomic_set(&skip_test, 0);
336 return;
337 }
338 cpu_relax();
339 }
340
341
342
343
344 atomic_inc(&start_count);
345
346 check_tsc_warp(loop_timeout(cpu));
347
348 while (atomic_read(&stop_count) != cpus-1)
349 cpu_relax();
350
351
352
353
354
355
356 if (!nr_warps) {
357 atomic_set(&test_runs, 0);
358
359 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
360 smp_processor_id(), cpu);
361
362 } else if (atomic_dec_and_test(&test_runs) || random_warps) {
363
364 atomic_set(&test_runs, 0);
365
366 pr_warn("TSC synchronization [CPU#%d -> CPU#%d]:\n",
367 smp_processor_id(), cpu);
368 pr_warn("Measured %Ld cycles TSC warp between CPUs, "
369 "turning off TSC clock.\n", max_warp);
370 if (random_warps)
371 pr_warn("TSC warped randomly between CPUs\n");
372 mark_tsc_unstable("check_tsc_sync_source failed");
373 }
374
375
376
377
378 atomic_set(&start_count, 0);
379 random_warps = 0;
380 nr_warps = 0;
381 max_warp = 0;
382 last_tsc = 0;
383
384
385
386
387 atomic_inc(&stop_count);
388
389
390
391
392 if (atomic_read(&test_runs) > 0)
393 goto retry;
394}
395
396
397
398
399void check_tsc_sync_target(void)
400{
401 struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust);
402 unsigned int cpu = smp_processor_id();
403 cycles_t cur_max_warp, gbl_max_warp;
404 int cpus = 2;
405
406
407 if (unsynchronized_tsc())
408 return;
409
410
411
412
413
414
415
416
417
418
419 if (tsc_store_and_check_tsc_adjust(false) || tsc_clocksource_reliable) {
420 atomic_inc(&skip_test);
421 return;
422 }
423
424retry:
425
426
427
428
429 atomic_inc(&start_count);
430 while (atomic_read(&start_count) != cpus)
431 cpu_relax();
432
433 cur_max_warp = check_tsc_warp(loop_timeout(cpu));
434
435
436
437
438 gbl_max_warp = max_warp;
439
440
441
442
443 atomic_inc(&stop_count);
444
445
446
447
448 while (atomic_read(&stop_count) != cpus)
449 cpu_relax();
450
451
452
453
454 atomic_set(&stop_count, 0);
455
456
457
458
459
460
461 if (!atomic_read(&test_runs))
462 return;
463
464
465
466
467
468
469 if (!cur_max_warp)
470 cur_max_warp = -gbl_max_warp;
471
472
473
474
475
476
477
478
479
480
481
482
483 cur->adjusted += cur_max_warp;
484
485 pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n",
486 cpu, cur_max_warp, cur->adjusted);
487
488 wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted);
489 goto retry;
490
491}
492
493#endif
494