blob: 9151f0ce6a428b934fcdd5fe1c0d1fd2f5dc79be [file] [log] [blame]
Thomas Gleixner250c2272007-10-11 11:17:24 +02001/*
Dave Jones835c34a2007-10-12 21:10:53 -04002 * check TSC synchronization.
Thomas Gleixner250c2272007-10-11 11:17:24 +02003 *
4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
5 *
6 * We check whether all boot CPUs have their TSC's synchronized,
7 * print a warning if not and turn off the TSC clock-source.
8 *
9 * The warp-check is point-to-point between two CPUs, the CPU
10 * initiating the bootup is the 'source CPU', the freshly booting
11 * CPU is the 'target CPU'.
12 *
13 * Only two CPUs may participate - they can enter in any order.
14 * ( The serial nature of the boot logic and the CPU hotplug lock
15 * protects against more than 2 CPUs entering this code. )
16 */
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000017#include <linux/topology.h>
Thomas Gleixner250c2272007-10-11 11:17:24 +020018#include <linux/spinlock.h>
19#include <linux/kernel.h>
Thomas Gleixner250c2272007-10-11 11:17:24 +020020#include <linux/smp.h>
21#include <linux/nmi.h>
22#include <asm/tsc.h>
23
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000024struct tsc_adjust {
Thomas Gleixner1d0095f2016-11-19 13:47:37 +000025 s64 bootval;
26 s64 adjusted;
27 unsigned long nextcheck;
28 bool warned;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000029};
30
31static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust);
32
Thomas Gleixner6a369582016-12-13 13:14:17 +000033void tsc_verify_tsc_adjust(bool resume)
Thomas Gleixner1d0095f2016-11-19 13:47:37 +000034{
35 struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust);
36 s64 curval;
37
38 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
39 return;
40
41 /* Rate limit the MSR check */
Thomas Gleixner6a369582016-12-13 13:14:17 +000042 if (!resume && time_before(jiffies, adj->nextcheck))
Thomas Gleixner1d0095f2016-11-19 13:47:37 +000043 return;
44
45 adj->nextcheck = jiffies + HZ;
46
47 rdmsrl(MSR_IA32_TSC_ADJUST, curval);
48 if (adj->adjusted == curval)
49 return;
50
51 /* Restore the original value */
52 wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted);
53
Thomas Gleixner6a369582016-12-13 13:14:17 +000054 if (!adj->warned || resume) {
Thomas Gleixner1d0095f2016-11-19 13:47:37 +000055 pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n",
56 smp_processor_id(), adj->adjusted, curval);
57 adj->warned = true;
58 }
59}
60
Thomas Gleixner5bae1562016-12-13 13:14:17 +000061static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval,
62 unsigned int cpu, bool bootcpu)
63{
64 /*
65 * First online CPU in a package stores the boot value in the
66 * adjustment value. This value might change later via the sync
67 * mechanism. If that fails we still can yell about boot values not
68 * being consistent.
69 *
70 * On the boot cpu we just force set the ADJUST value to 0 if it's
71 * non zero. We don't do that on non boot cpus because physical
72 * hotplug should have set the ADJUST register to a value > 0 so
73 * the TSC is in sync with the already running cpus.
74 *
75 * But we always force positive ADJUST values. Otherwise the TSC
76 * deadline timer creates an interrupt storm. Sigh!
77 */
78 if ((bootcpu && bootval != 0) || (!bootcpu && bootval < 0)) {
79 pr_warn("TSC ADJUST: CPU%u: %lld force to 0\n", cpu, bootval);
80 wrmsrl(MSR_IA32_TSC_ADJUST, 0);
81 bootval = 0;
82 }
83 cur->adjusted = bootval;
84}
85
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000086#ifndef CONFIG_SMP
Thomas Gleixner5bae1562016-12-13 13:14:17 +000087bool __init tsc_store_and_check_tsc_adjust(bool bootcpu)
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000088{
Thomas Gleixnerb8365542016-11-29 20:28:31 +010089 struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust);
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000090 s64 bootval;
91
92 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
Thomas Gleixnera36f5132016-11-19 13:47:39 +000093 return false;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +000094
95 rdmsrl(MSR_IA32_TSC_ADJUST, bootval);
96 cur->bootval = bootval;
Thomas Gleixner1d0095f2016-11-19 13:47:37 +000097 cur->nextcheck = jiffies + HZ;
Thomas Gleixner5bae1562016-12-13 13:14:17 +000098 tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu);
Thomas Gleixnera36f5132016-11-19 13:47:39 +000099 return false;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000100}
101
102#else /* !CONFIG_SMP */
103
104/*
105 * Store and check the TSC ADJUST MSR if available
106 */
Thomas Gleixner5bae1562016-12-13 13:14:17 +0000107bool tsc_store_and_check_tsc_adjust(bool bootcpu)
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000108{
109 struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust);
110 unsigned int refcpu, cpu = smp_processor_id();
Thomas Gleixner31f8a652016-12-01 13:26:58 +0100111 struct cpumask *mask;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000112 s64 bootval;
113
114 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000115 return false;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000116
117 rdmsrl(MSR_IA32_TSC_ADJUST, bootval);
118 cur->bootval = bootval;
Thomas Gleixner1d0095f2016-11-19 13:47:37 +0000119 cur->nextcheck = jiffies + HZ;
120 cur->warned = false;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000121
122 /*
123 * Check whether this CPU is the first in a package to come up. In
124 * this case do not check the boot value against another package
Thomas Gleixner5bae1562016-12-13 13:14:17 +0000125 * because the new package might have been physically hotplugged,
126 * where TSC_ADJUST is expected to be different. When called on the
127 * boot CPU topology_core_cpumask() might not be available yet.
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000128 */
Thomas Gleixner31f8a652016-12-01 13:26:58 +0100129 mask = topology_core_cpumask(cpu);
130 refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000131
132 if (refcpu >= nr_cpu_ids) {
Thomas Gleixner5bae1562016-12-13 13:14:17 +0000133 tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(),
134 bootcpu);
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000135 return false;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000136 }
137
138 ref = per_cpu_ptr(&tsc_adjust, refcpu);
139 /*
140 * Compare the boot value and complain if it differs in the
141 * package.
142 */
143 if (bootval != ref->bootval) {
144 pr_warn("TSC ADJUST differs: Reference CPU%u: %lld CPU%u: %lld\n",
145 refcpu, ref->bootval, cpu, bootval);
146 }
147 /*
148 * The TSC_ADJUST values in a package must be the same. If the boot
149 * value on this newly upcoming CPU differs from the adjustment
150 * value of the already online CPU in this package, set it to that
151 * adjusted value.
152 */
153 if (bootval != ref->adjusted) {
154 pr_warn("TSC ADJUST synchronize: Reference CPU%u: %lld CPU%u: %lld\n",
155 refcpu, ref->adjusted, cpu, bootval);
156 cur->adjusted = ref->adjusted;
157 wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted);
158 }
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000159 /*
160 * We have the TSCs forced to be in sync on this package. Skip sync
161 * test:
162 */
163 return true;
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000164}
165
Thomas Gleixner250c2272007-10-11 11:17:24 +0200166/*
167 * Entry/exit counters that make sure that both CPUs
168 * run the measurement code at once:
169 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400170static atomic_t start_count;
171static atomic_t stop_count;
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000172static atomic_t skip_test;
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000173static atomic_t test_runs;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200174
175/*
176 * We use a raw spinlock in this exceptional case, because
177 * we want to have the fastest, inlined, non-debug version
178 * of a critical section, to be able to prove TSC time-warps:
179 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400180static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Ingo Molnar643bec92009-05-07 09:12:50 +0200181
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400182static cycles_t last_tsc;
183static cycles_t max_warp;
184static int nr_warps;
Thomas Gleixnerbec85202016-11-19 13:47:35 +0000185static int random_warps;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200186
187/*
Andy Lutomirskieee69462015-06-25 18:44:09 +0200188 * TSC-warp measurement loop running on both CPUs. This is not called
189 * if there is no TSC.
Thomas Gleixner250c2272007-10-11 11:17:24 +0200190 */
Thomas Gleixner76d3b85152016-11-19 13:47:41 +0000191static cycles_t check_tsc_warp(unsigned int timeout)
Thomas Gleixner250c2272007-10-11 11:17:24 +0200192{
Thomas Gleixner76d3b85152016-11-19 13:47:41 +0000193 cycles_t start, now, prev, end, cur_max_warp = 0;
Thomas Gleixnerbec85202016-11-19 13:47:35 +0000194 int i, cur_warps = 0;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200195
Andy Lutomirskieee69462015-06-25 18:44:09 +0200196 start = rdtsc_ordered();
Thomas Gleixner250c2272007-10-11 11:17:24 +0200197 /*
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800198 * The measurement runs for 'timeout' msecs:
Thomas Gleixner250c2272007-10-11 11:17:24 +0200199 */
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800200 end = start + (cycles_t) tsc_khz * timeout;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200201 now = start;
202
203 for (i = 0; ; i++) {
204 /*
205 * We take the global lock, measure TSC, save the
206 * previous TSC that was measured (possibly on
207 * another CPU) and update the previous TSC timestamp.
208 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100209 arch_spin_lock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +0200210 prev = last_tsc;
Andy Lutomirskieee69462015-06-25 18:44:09 +0200211 now = rdtsc_ordered();
Thomas Gleixner250c2272007-10-11 11:17:24 +0200212 last_tsc = now;
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100213 arch_spin_unlock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +0200214
215 /*
216 * Be nice every now and then (and also check whether
Ingo Molnardf435102008-01-30 13:33:23 +0100217 * measurement is done [we also insert a 10 million
Thomas Gleixner250c2272007-10-11 11:17:24 +0200218 * loops safety exit, so we dont lock up in case the
219 * TSC readout is totally broken]):
220 */
221 if (unlikely(!(i & 7))) {
Ingo Molnardf435102008-01-30 13:33:23 +0100222 if (now > end || i > 10000000)
Thomas Gleixner250c2272007-10-11 11:17:24 +0200223 break;
224 cpu_relax();
225 touch_nmi_watchdog();
226 }
227 /*
228 * Outside the critical section we can now see whether
229 * we saw a time-warp of the TSC going backwards:
230 */
231 if (unlikely(prev > now)) {
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100232 arch_spin_lock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +0200233 max_warp = max(max_warp, prev - now);
Thomas Gleixner76d3b85152016-11-19 13:47:41 +0000234 cur_max_warp = max_warp;
Thomas Gleixnerbec85202016-11-19 13:47:35 +0000235 /*
236 * Check whether this bounces back and forth. Only
237 * one CPU should observe time going backwards.
238 */
239 if (cur_warps != nr_warps)
240 random_warps++;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200241 nr_warps++;
Thomas Gleixnerbec85202016-11-19 13:47:35 +0000242 cur_warps = nr_warps;
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100243 arch_spin_unlock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +0200244 }
Ingo Molnarad8ca492008-01-30 13:33:24 +0100245 }
Arjan van de Venbde78a72008-07-08 09:51:56 -0700246 WARN(!(now-start),
247 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
Ingo Molnarad8ca492008-01-30 13:33:24 +0100248 now-start, end-start);
Thomas Gleixner76d3b85152016-11-19 13:47:41 +0000249 return cur_max_warp;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200250}
251
252/*
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800253 * If the target CPU coming online doesn't have any of its core-siblings
254 * online, a timeout of 20msec will be used for the TSC-warp measurement
255 * loop. Otherwise a smaller timeout of 2msec will be used, as we have some
256 * information about this socket already (and this information grows as we
257 * have more and more logical-siblings in that socket).
258 *
259 * Ideally we should be able to skip the TSC sync check on the other
260 * core-siblings, if the first logical CPU in a socket passed the sync test.
261 * But as the TSC is per-logical CPU and can potentially be modified wrongly
262 * by the bios, TSC sync test for smaller duration should be able
263 * to catch such errors. Also this will catch the condition where all the
264 * cores in the socket doesn't get reset at the same time.
265 */
266static inline unsigned int loop_timeout(int cpu)
267{
Bartosz Golaszewski7d79a7b2015-05-26 15:11:35 +0200268 return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20;
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800269}
270
271/*
Thomas Gleixner250c2272007-10-11 11:17:24 +0200272 * Source CPU calls into this - it waits for the freshly booted
273 * target CPU to arrive and then starts the measurement:
274 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400275void check_tsc_sync_source(int cpu)
Thomas Gleixner250c2272007-10-11 11:17:24 +0200276{
277 int cpus = 2;
278
279 /*
280 * No need to check if we already know that the TSC is not
Andy Lutomirskieee69462015-06-25 18:44:09 +0200281 * synchronized or if we have no TSC.
Thomas Gleixner250c2272007-10-11 11:17:24 +0200282 */
283 if (unsynchronized_tsc())
284 return;
285
Suresh Siddha28a00182011-11-04 15:42:17 -0700286 if (tsc_clocksource_reliable) {
Mike Travis9b3660a2009-11-17 18:22:16 -0600287 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
288 pr_info(
289 "Skipped synchronization checks as TSC is reliable.\n");
Alok Katariaeca0cd02008-10-31 12:01:58 -0700290 return;
291 }
292
Thomas Gleixner250c2272007-10-11 11:17:24 +0200293 /*
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000294 * Set the maximum number of test runs to
295 * 1 if the CPU does not provide the TSC_ADJUST MSR
296 * 3 if the MSR is available, so the target can try to adjust
297 */
298 if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
299 atomic_set(&test_runs, 1);
300 else
301 atomic_set(&test_runs, 3);
302retry:
303 /*
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000304 * Wait for the target to start or to skip the test:
Thomas Gleixner250c2272007-10-11 11:17:24 +0200305 */
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000306 while (atomic_read(&start_count) != cpus - 1) {
307 if (atomic_read(&skip_test) > 0) {
308 atomic_set(&skip_test, 0);
309 return;
310 }
Thomas Gleixner250c2272007-10-11 11:17:24 +0200311 cpu_relax();
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000312 }
313
Thomas Gleixner250c2272007-10-11 11:17:24 +0200314 /*
315 * Trigger the target to continue into the measurement too:
316 */
317 atomic_inc(&start_count);
318
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800319 check_tsc_warp(loop_timeout(cpu));
Thomas Gleixner250c2272007-10-11 11:17:24 +0200320
321 while (atomic_read(&stop_count) != cpus-1)
322 cpu_relax();
323
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000324 /*
325 * If the test was successful set the number of runs to zero and
326 * stop. If not, decrement the number of runs an check if we can
327 * retry. In case of random warps no retry is attempted.
328 */
329 if (!nr_warps) {
330 atomic_set(&test_runs, 0);
331
332 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
333 smp_processor_id(), cpu);
334
335 } else if (atomic_dec_and_test(&test_runs) || random_warps) {
336 /* Force it to 0 if random warps brought us here */
337 atomic_set(&test_runs, 0);
338
Mike Travis9b3660a2009-11-17 18:22:16 -0600339 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
340 smp_processor_id(), cpu);
Ingo Molnar643bec92009-05-07 09:12:50 +0200341 pr_warning("Measured %Ld cycles TSC warp between CPUs, "
342 "turning off TSC clock.\n", max_warp);
Thomas Gleixnerbec85202016-11-19 13:47:35 +0000343 if (random_warps)
344 pr_warning("TSC warped randomly between CPUs\n");
Thomas Gleixner250c2272007-10-11 11:17:24 +0200345 mark_tsc_unstable("check_tsc_sync_source failed");
Thomas Gleixner250c2272007-10-11 11:17:24 +0200346 }
347
348 /*
Mike Galbraith4c6b8b42008-01-30 13:30:04 +0100349 * Reset it - just in case we boot another CPU later:
350 */
351 atomic_set(&start_count, 0);
Thomas Gleixnerbec85202016-11-19 13:47:35 +0000352 random_warps = 0;
Mike Galbraith4c6b8b42008-01-30 13:30:04 +0100353 nr_warps = 0;
354 max_warp = 0;
355 last_tsc = 0;
356
357 /*
Thomas Gleixner250c2272007-10-11 11:17:24 +0200358 * Let the target continue with the bootup:
359 */
360 atomic_inc(&stop_count);
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000361
362 /*
363 * Retry, if there is a chance to do so.
364 */
365 if (atomic_read(&test_runs) > 0)
366 goto retry;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200367}
368
369/*
370 * Freshly booted CPUs call into this:
371 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400372void check_tsc_sync_target(void)
Thomas Gleixner250c2272007-10-11 11:17:24 +0200373{
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000374 struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust);
375 unsigned int cpu = smp_processor_id();
376 cycles_t cur_max_warp, gbl_max_warp;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200377 int cpus = 2;
378
Andy Lutomirskieee69462015-06-25 18:44:09 +0200379 /* Also aborts if there is no TSC. */
Suresh Siddha28a00182011-11-04 15:42:17 -0700380 if (unsynchronized_tsc() || tsc_clocksource_reliable)
Thomas Gleixner250c2272007-10-11 11:17:24 +0200381 return;
382
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000383 /*
384 * Store, verify and sanitize the TSC adjust register. If
385 * successful skip the test.
386 */
Thomas Gleixner5bae1562016-12-13 13:14:17 +0000387 if (tsc_store_and_check_tsc_adjust(false)) {
Thomas Gleixnera36f5132016-11-19 13:47:39 +0000388 atomic_inc(&skip_test);
389 return;
390 }
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000391
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000392retry:
Thomas Gleixner250c2272007-10-11 11:17:24 +0200393 /*
394 * Register this CPU's participation and wait for the
395 * source CPU to start the measurement:
396 */
397 atomic_inc(&start_count);
398 while (atomic_read(&start_count) != cpus)
399 cpu_relax();
400
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000401 cur_max_warp = check_tsc_warp(loop_timeout(cpu));
402
403 /*
404 * Store the maximum observed warp value for a potential retry:
405 */
406 gbl_max_warp = max_warp;
Thomas Gleixner250c2272007-10-11 11:17:24 +0200407
408 /*
409 * Ok, we are done:
410 */
411 atomic_inc(&stop_count);
412
413 /*
414 * Wait for the source CPU to print stuff:
415 */
416 while (atomic_read(&stop_count) != cpus)
417 cpu_relax();
Thomas Gleixner4c5e3c62016-11-19 13:47:40 +0000418
419 /*
420 * Reset it for the next sync test:
421 */
422 atomic_set(&stop_count, 0);
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000423
424 /*
425 * Check the number of remaining test runs. If not zero, the test
426 * failed and a retry with adjusted TSC is possible. If zero the
427 * test was either successful or failed terminally.
428 */
429 if (!atomic_read(&test_runs))
430 return;
431
432 /*
433 * If the warp value of this CPU is 0, then the other CPU
434 * observed time going backwards so this TSC was ahead and
435 * needs to move backwards.
436 */
437 if (!cur_max_warp)
438 cur_max_warp = -gbl_max_warp;
439
440 /*
441 * Add the result to the previous adjustment value.
442 *
443 * The adjustement value is slightly off by the overhead of the
444 * sync mechanism (observed values are ~200 TSC cycles), but this
445 * really depends on CPU, node distance and frequency. So
446 * compensating for this is hard to get right. Experiments show
447 * that the warp is not longer detectable when the observed warp
448 * value is used. In the worst case the adjustment needs to go
449 * through a 3rd run for fine tuning.
Thomas Gleixner5bae1562016-12-13 13:14:17 +0000450 *
451 * But we must make sure that the value doesn't become negative
452 * otherwise TSC deadline timer will create an interrupt storm.
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000453 */
454 cur->adjusted += cur_max_warp;
Thomas Gleixner5bae1562016-12-13 13:14:17 +0000455 if (cur->adjusted < 0)
456 cur->adjusted = 0;
Thomas Gleixnercc4db262016-11-19 13:47:43 +0000457
458 pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n",
459 cpu, cur_max_warp, cur->adjusted);
460
461 wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted);
462 goto retry;
463
Thomas Gleixner250c2272007-10-11 11:17:24 +0200464}
Thomas Gleixner8b223bc2016-11-19 13:47:36 +0000465
466#endif /* CONFIG_SMP */