github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM x86 timekeeping fixes and cleanups for 7.4

Browse Source 
Fix a variety of warts in KVM's handling of guest TSC, kvmclock, and
 timekeeping in general, as a (big) first step towards addressing several
 longstanding problems with the kvmclock implementation.
 
 Most notably, fix a precision issue where get_kvmclock_ns() computed the
 clock from the *host* TSC without applying guest TSC scaling, causing
 systemic drift from the values the guest computes from its own TSC.
 
 As a bonus cleanup, rip out the whole pvclock_gtod_data hack, which was
 shadowing the kernel's timekeeping, and use ktime snapshots.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmrRhTwACgkQOlYIJqCj
 N/3Aqw/+N5/BczVl7Mh2YuxzYcxPtaOroh99ZHdwDMiEh3ok3RbMejGREnvte8rF
 IIAFQMDy7sr9dqI74RbkjLUaOrFteoCGvPNsEj+lU6mdS3/u6cCYg6EYKS0Ao34e
 AQ9jM+zAjopxLLfD8cBwiSNdiH3oG83v+OodeZiQuG/z+n9rDEFXqrCySgT736kW
 hW6NAB60RUvX2nGNqlE7o/HF1/FlOq0pHuZFwQlc6eP6RyQd101m39qyBaopURss
 oNExGDaGMYY8AwKMCj1041fpKMA0VTqaiReMMFc/9moSXuLT1Fl5xqCEn7kKkOqz
 g8nMRXghSXslr+hroTbE7JDE3WijlbsGFcW2NrFW0HfZbcCiZ7boVuXmGrWpX8GR
 kqpzgazAuczPJRvGhzJgMAovo1jrODfzjlbkT+QY4inqaBiXg4hP8pZB2cldwsmx
 Paxh5PuKjgLGYJwTidmoEMHmRcUaNd+bi6xDQnEOqJgdOGtB+8teH7drN9JnhusM
 LRerrfMTaFDK3bk2FFJwYhR2on/TQ1gQQjATKxeDFZwyfTPYMas30hyOgR02aYSq
 NKxXCYCp2sOhjYIkaPYmZQ8U1+4Zwp9e6pCMcUyopTe2viM/jILFD71Aw1CZZvGe
 WFwP4tGve2kUXGmbuxwBDWlknVEAM2XBMLAcz3S1yWhxme8o8KI=
 =AwkA
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-clocks-7.4' of https://github.com/kvm-x86/linux into HEAD

KVM x86 timekeeping fixes and cleanups for 7.4

Fix a variety of warts in KVM's handling of guest TSC, kvmclock, and
timekeeping in general, as a (big) first step towards addressing several
longstanding problems with the kvmclock implementation.

Most notably, fix a precision issue where get_kvmclock_ns() computed the
clock from the *host* TSC without applying guest TSC scaling, causing
systemic drift from the values the guest computes from its own TSC.

As a bonus cleanup, rip out the whole pvclock_gtod_data hack, which was
shadowing the kernel's timekeeping, and use ktime snapshots.
This commit is contained in:
Paolo Bonzini 2026-10-16 09:46:35 +02:00
commit e853450f17
7 changed files with 238 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
arch/x86/include/asm/kvm_host.h
View File

@ -870,7 +870,7 @@ struct kvm_vcpu_arch {
gpa_t time;
s8 pvclock_tsc_shift;
u32 pvclock_tsc_mul;
unsigned int hw_tsc_khz;
u64 hw_tsc_hz;
struct gfn_to_pfn_cache pv_time;
/* set guest stopped flag in pvclock flags field */
bool pvclock_set_guest_stopped_request;
@ -1242,6 +1242,7 @@ struct kvm_arch {
u64 last_tsc_write;
u32 last_tsc_khz;
u64 last_tsc_offset;
u64 last_tsc_scaling_ratio;
u64 cur_tsc_nsec;
u64 cur_tsc_write;
u64 cur_tsc_offset;
@ -1257,6 +1258,9 @@ struct kvm_arch {
u64 master_kernel_ns;
u64 master_cycle_now;
struct ratelimit_state kvmclock_update_rs;
u64 master_tsc_scaling_ratio;
s8 master_tsc_shift;
u32 master_tsc_mul;
#ifdef CONFIG_KVM_HYPERV
struct kvm_hv hyperv;

1
arch/x86/kvm/cpuid.c
View File

@ -16,6 +16,7 @@
#include <linux/vmalloc.h>
#include <linux/uaccess.h>
#include <linux/sched/stat.h>
#include <linux/units.h>
#include <asm/processor.h>
#include <asm/user.h>

3
arch/x86/kvm/msrs.c
View File

@ -1631,7 +1631,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
if (msr_info->host_initiated) {
kvm_synchronize_tsc(vcpu, &data);
} else if (!vcpu->arch.guest_tsc_protected) {
u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) -
vcpu->arch.l1_tsc_offset;
adjust_tsc_offset_guest(vcpu, adj);
vcpu->arch.ia32_tsc_adjust_msr += adj;
}

3
arch/x86/kvm/svm/svm.c
View File

@ -5617,7 +5617,8 @@ static __init int svm_hardware_setup(void)
XFEATURE_MASK_BNDCSR);
if (tsc_scaling) {
if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR) ||
!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
tsc_scaling = false;
} else {
pr_info("TSC scaling supported\n");

10
arch/x86/kvm/vmx/vmx.c
View File

@ -2843,6 +2843,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf,
if (!cpu_has_sgx())
_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING;
/*
* KVM doesn't re-derive the TSC scaling ratio when the host TSC
* frequency changes, so TSC scaling is only usable with a constant
* TSC. Clear the control here rather than in vmx_hardware_setup() so
* that the per-CPU configs recomputed by vmx_check_processor_compat()
* stay consistent with the golden vmcs_config.
*/
if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING;
if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS)
_cpu_based_3rd_exec_control =
adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL,

476
arch/x86/kvm/x86.c
View File

@ -35,6 +35,7 @@
#include "smm.h"
#include <linux/clocksource.h>
#include <linux/timekeeping.h>
#include <linux/interrupt.h>
#include <linux/kvm.h>
#include <linux/fs.h>
@ -61,6 +62,7 @@
#include <linux/mem_encrypt.h>
#include <linux/suspend.h>
#include <linux/smp.h>
#include <linux/units.h>
#include <trace/events/ipi.h>
#include <trace/events/kvm.h>
@ -925,19 +927,13 @@ static void update_pvclock_gtod(struct timekeeper *tk)
write_seqcount_end(&vdata->seq);
}
#endif
static s64 get_kvmclock_base_ns(void)
{
/* Count up from boot time, but with the frequency of the raw clock. */
return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT));
}
#else
static s64 get_kvmclock_base_ns(void)
{
/* Master clock not used, so we can just use CLOCK_BOOTTIME. */
return ktime_get_boottime_ns();
}
#endif
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
@ -945,32 +941,57 @@ static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
return dividend;
}
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz,
s8 *pshift, u32 *pmultiplier)
{
uint64_t scaled64;
int32_t shift = 0;
uint64_t tps64;
uint32_t tps32;
u64 scaled_hz_u64 = scaled_hz;
s32 shift = 0;
u64 base_hz_u64;
u32 base32;
tps64 = base_hz;
scaled64 = scaled_hz;
while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
tps64 >>= 1;
/*
* This function calculates a fixed-point multiplier and shift such
* that:
* time_ns = (tsc_cycles << shift) * multiplier >> 32
*
* Where tsc_cycles tick at base_hz, and time_ns should count at
* scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion).
*
* The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift
* to keep everything in range.
*/
base_hz_u64 = base_hz;
/*
* Start by shifting base_hz right until it fits in 32 bits, and
* is lower than double the target rate. This introduces a negative
* shift value which would result in pvclock_scale_delta() shifting
* the actual tick count right before performing the multiplication.
*/
while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) {
base_hz_u64 >>= 1;
shift--;
}
tps32 = (uint32_t)tps64;
while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
scaled64 >>= 1;
/* Now the shifted base_hz fits in 32 bits. */
base32 = (u32)base_hz_u64;
/*
* Next, shift scaled_hz right until it fits in 32 bits, and ensure
* that the shifted base_hz is strictly larger (so that the result of the
* final division also fits in 32 bits).
*/
while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) {
if (scaled_hz_u64 >> 32 || base32 & BIT(31))
scaled_hz_u64 >>= 1;
else
tps32 <<= 1;
base32 <<= 1;
shift++;
}
*pshift = shift;
*pmultiplier = div_frac(scaled64, tps32);
*pmultiplier = div_frac(scaled_hz_u64, base32);
}
#ifdef CONFIG_X86_64
@ -1061,11 +1082,15 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
vcpu->arch.virtual_tsc_mult,
vcpu->arch.virtual_tsc_shift);
tsc += vcpu->arch.this_tsc_write;
return tsc;
s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec;
u64 tsc;
/* Handle negative deltas gracefully (master clock ref may be earlier) */
tsc = pvclock_scale_delta(abs(delta_ns),
vcpu->arch.virtual_tsc_mult,
vcpu->arch.virtual_tsc_shift);
return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc);
}
#ifdef CONFIG_X86_64
@ -1131,11 +1156,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio)
return _tsc;
}
u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc,
u64 target_tsc)
{
u64 tsc;
tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
return target_tsc - tsc;
}
@ -1254,6 +1280,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
kvm->arch.last_tsc_write = tsc;
kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
kvm->arch.last_tsc_offset = offset;
kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
vcpu->arch.last_guest_tsc = tsc;
@ -1296,7 +1323,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
bool synchronizing = false;
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
offset = kvm_compute_l1_tsc_offset(vcpu, data);
offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data);
ns = get_kvmclock_base_ns();
elapsed = ns - kvm->arch.last_tsc_nsec;
@ -1345,7 +1372,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
} else {
u64 delta = nsec_to_cycles(vcpu, elapsed);
data += delta;
offset = kvm_compute_l1_tsc_offset(vcpu, data);
offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data);
}
matched = true;
}
@ -1356,126 +1383,27 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
#ifdef CONFIG_X86_64
static u64 read_tsc(void)
static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap,
u64 *tsc_timestamp)
{
u64 ret = (u64)rdtsc_ordered();
u64 last = pvclock_gtod_data.clock.cycle_last;
if (likely(ret >= last))
return ret;
/*
* GCC likes to generate cmov here, but this branch is extremely
* predictable (it's just a function of time and the likely is
* very likely) and there's a data dependence, so force GCC
* to generate a branch instead. I don't barrier() because
* we don't actually need a barrier, and if this function
* ever gets inlined it will generate worse code.
* ktime_get_snapshot_id() cannot fail for standard clock IDs
* (only for invalid/aux clocks or during suspend, with a WARN).
*/
asm volatile ("");
return last;
}
if (!snap->valid)
return false;
static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
int *mode)
{
u64 tsc_pg_val;
long v;
switch (clock->vclock_mode) {
case VDSO_CLOCKMODE_HVCLOCK:
if (hv_read_tsc_page_tsc(hv_get_tsc_page(),
tsc_timestamp, &tsc_pg_val)) {
/* TSC page valid */
*mode = VDSO_CLOCKMODE_HVCLOCK;
v = (tsc_pg_val - clock->cycle_last) &
clock->mask;
} else {
/* TSC page invalid */
*mode = VDSO_CLOCKMODE_NONE;
}
break;
case VDSO_CLOCKMODE_TSC:
*mode = VDSO_CLOCKMODE_TSC;
*tsc_timestamp = read_tsc();
v = (*tsc_timestamp - clock->cycle_last) &
clock->mask;
break;
default:
*mode = VDSO_CLOCKMODE_NONE;
if (snap->cs_id == CSID_X86_TSC) {
*tsc_timestamp = snap->cycles;
return true;
}
if (*mode == VDSO_CLOCKMODE_NONE)
*tsc_timestamp = v = 0;
if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) {
*tsc_timestamp = snap->hw_cycles;
return true;
}
return v * clock->mult;
}
/*
* As with get_kvmclock_base_ns(), this counts from boot time, at the
* frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot).
*/
static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp)
{
struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
unsigned long seq;
int mode;
u64 ns;
do {
seq = read_seqcount_begin(&gtod->seq);
ns = gtod->raw_clock.base_cycles;
ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
ns >>= gtod->raw_clock.shift;
ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
*t = ns;
return mode;
}
/*
* This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with
* no boot time offset.
*/
static int do_monotonic(s64 *t, u64 *tsc_timestamp)
{
struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
unsigned long seq;
int mode;
u64 ns;
do {
seq = read_seqcount_begin(&gtod->seq);
ns = gtod->clock.base_cycles;
ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
ns >>= gtod->clock.shift;
ns += ktime_to_ns(gtod->clock.offset);
} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
*t = ns;
return mode;
}
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
{
struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
unsigned long seq;
int mode;
u64 ns;
do {
seq = read_seqcount_begin(&gtod->seq);
ts->tv_sec = gtod->wall_time_sec;
ns = gtod->clock.base_cycles;
ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
ns >>= gtod->clock.shift;
} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
return mode;
return false;
}
/*
@ -1485,12 +1413,14 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
*/
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
{
/* checked again under seqlock below */
if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
struct system_time_snapshot snap = {};
ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap);
if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
return false;
return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns,
tsc_timestamp));
*kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT));
return true;
}
/*
@ -1499,12 +1429,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
*/
bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
{
/* checked again under seqlock below */
if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
struct system_time_snapshot snap = {};
ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap);
if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
return false;
return gtod_is_based_on_tsc(do_monotonic(kernel_ns,
tsc_timestamp));
*kernel_ns = ktime_to_ns(snap.systime);
return true;
}
/*
@ -1517,11 +1449,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
u64 *tsc_timestamp)
{
/* checked again under seqlock below */
if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
struct system_time_snapshot snap = {};
ktime_get_snapshot_id(CLOCK_REALTIME, &snap);
if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
return false;
return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
*ts = ktime_to_timespec64(snap.systime);
return true;
}
#endif
@ -1566,6 +1501,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
*
*/
static unsigned long get_cpu_tsc_khz(void);
static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
@ -1589,9 +1526,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
&& !ka->backwards_tsc_observed
&& !ka->boot_vcpu_runs_old_kvmclock;
if (ka->use_master_clock)
if (ka->use_master_clock) {
u64 tsc_hz;
atomic_set(&kvm_guest_has_master_clock, 1);
/*
* Copy the scaling ratio and precompute the mul/shift for
* converting guest TSC to nanoseconds. These are used by
* get_kvmclock() to compute kvmclock from the host TSC
* without needing a vCPU reference.
*/
ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio;
tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ;
if (tsc_hz && kvm_caps.has_tsc_control)
tsc_hz = kvm_scale_tsc(tsc_hz,
ka->master_tsc_scaling_ratio);
if (tsc_hz)
kvm_get_time_scale(NSEC_PER_SEC, tsc_hz,
&ka->master_tsc_shift,
&ka->master_tsc_mul);
else
ka->use_master_clock = false;
}
vclock_mode = pvclock_gtod_data.clock.vclock_mode;
trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
vcpus_matched);
@ -1658,39 +1616,49 @@ static unsigned long get_cpu_tsc_khz(void)
}
/* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */
static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
static bool __get_kvmclock_master_clock(struct kvm *kvm,
struct kvm_clock_data *data)
{
#ifdef CONFIG_X86_64
struct kvm_arch *ka = &kvm->arch;
struct pvclock_vcpu_time_info hv_clock;
struct timespec64 ts;
/* both __this_cpu_read() and rdtsc() should be on the same cpu */
get_cpu();
if (!ka->use_master_clock)
return false;
data->flags = 0;
if (ka->use_master_clock &&
(cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) {
#ifdef CONFIG_X86_64
struct timespec64 ts;
if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc))
return false;
if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
} else
#endif
data->host_tsc = rdtsc();
data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC |
KVM_CLOCK_TSC_STABLE;
data->flags |= KVM_CLOCK_TSC_STABLE;
hv_clock.tsc_timestamp = ka->master_cycle_now;
hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL,
&hv_clock.tsc_shift,
&hv_clock.tsc_to_system_mul);
data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
hv_clock.tsc_timestamp = ka->master_cycle_now;
hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
/*
* Use the precomputed guest-TSC-based mul/shift so that the kvmclock
* value matches what the guest computes from its own TSC.
*/
hv_clock.tsc_shift = ka->master_tsc_shift;
hv_clock.tsc_to_system_mul = ka->master_tsc_mul;
if (kvm_caps.has_tsc_control) {
u64 tsc_delta = data->host_tsc - ka->master_cycle_now;
tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio);
data->clock = hv_clock.system_time +
pvclock_scale_delta(tsc_delta,
hv_clock.tsc_to_system_mul,
hv_clock.tsc_shift);
} else {
data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
}
put_cpu();
return true;
#else
return false;
#endif
}
static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
@ -1699,8 +1667,11 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
unsigned seq;
do {
data->flags = 0;
seq = read_seqcount_begin(&ka->pvclock_sc);
__get_kvmclock(kvm, data);
if (!__get_kvmclock_master_clock(kvm, data))
data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
} while (read_seqcount_retry(&ka->pvclock_sc, seq));
}
@ -1762,36 +1733,45 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock,
int kvm_guest_time_update(struct kvm_vcpu *v)
{
u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns;
struct kvm_arch *ka __maybe_unused = &v->kvm->arch;
struct pvclock_vcpu_time_info hv_clock = {};
unsigned long flags, tgt_tsc_khz;
unsigned seq;
struct kvm_vcpu_arch *vcpu = &v->arch;
struct kvm_arch *ka = &v->kvm->arch;
s64 kernel_ns;
u64 tsc_timestamp, host_tsc;
bool use_master_clock;
kernel_ns = 0;
host_tsc = 0;
/*
* If the host uses TSC clock, then passthrough TSC as stable
* to the guest.
*/
#ifdef CONFIG_X86_64
bool use_master_clock;
unsigned int seq;
do {
seq = read_seqcount_begin(&ka->pvclock_sc);
use_master_clock = ka->use_master_clock;
if (use_master_clock) {
host_tsc = ka->master_cycle_now;
kernel_ns = ka->master_kernel_ns;
}
} while (read_seqcount_retry(&ka->pvclock_sc, seq));
if (!use_master_clock)
continue;
/* Keep irq disabled to prevent changes to the clock */
local_irq_save(flags);
tgt_tsc_khz = get_cpu_tsc_khz();
if (unlikely(tgt_tsc_khz == 0)) {
local_irq_restore(flags);
if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) {
use_master_clock = false;
continue;
}
master_tsc = ka->master_cycle_now;
master_ns = ka->master_kernel_ns;
} while (read_seqcount_retry(&ka->pvclock_sc, seq));
#else
const bool use_master_clock = false;
#endif
/*
* Ensure reading the TSC+frequency pair is done on the same CPU. When
* NOT using the master clock, the TSC frequency may vary between CPUs.
*/
preempt_disable();
tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ;
if (unlikely(tgt_tsc_hz == 0)) {
preempt_enable();
kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
return 1;
}
@ -1820,28 +1800,43 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
}
}
local_irq_restore(flags);
/*
* Refresh L1's last "observed" TSC to match the PV clock's timestamp,
* e.g. so that the guest can't see a TSC that's behind the reference.
*/
vcpu->last_guest_tsc = tsc_timestamp;
preempt_enable();
/* With all the info we got, fill in the values */
if (kvm_caps.has_tsc_control) {
tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz,
v->arch.l1_tsc_scaling_ratio);
tgt_tsc_khz = tgt_tsc_khz ? : 1;
tgt_tsc_hz = tgt_tsc_hz ? : 1;
}
if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) {
kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz,
&vcpu->pvclock_tsc_shift,
&vcpu->pvclock_tsc_mul);
vcpu->hw_tsc_khz = tgt_tsc_khz;
vcpu->hw_tsc_hz = tgt_tsc_hz;
}
hv_clock.tsc_shift = vcpu->pvclock_tsc_shift;
hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul;
hv_clock.tsc_timestamp = tsc_timestamp;
hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
vcpu->last_guest_tsc = tsc_timestamp;
/*
* If the master clock is NOT in use, the reference time placed in the
* hv_clock is "now". If master clock is in use, the reference time is
* the master clock's snapshot from some time in the past, not "now".
*/
if (use_master_clock) {
hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc);
hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset;
} else {
hv_clock.tsc_timestamp = tsc_timestamp;
hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
}
/* If the host uses TSC clocksource, then it is stable */
hv_clock.flags = 0;
@ -1903,63 +1898,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
* wallclock and kvmclock times, and subtracting one from the other.
*
* Fall back to using their values at slightly different moments by
* calling ktime_get_real_ns() and get_kvmclock_ns() separately.
* calling ktime_get_real_ns() and get_kvmclock() separately.
*/
uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
struct pvclock_vcpu_time_info hv_clock;
struct kvm_arch *ka = &kvm->arch;
unsigned long seq, local_tsc_khz;
struct timespec64 ts;
uint64_t host_tsc;
struct kvm_clock_data data;
do {
seq = read_seqcount_begin(&ka->pvclock_sc);
local_tsc_khz = 0;
if (!ka->use_master_clock)
break;
/*
* The TSC read and the call to get_cpu_tsc_khz() must happen
* on the same CPU.
*/
get_cpu();
local_tsc_khz = get_cpu_tsc_khz();
if (local_tsc_khz &&
!kvm_get_walltime_and_clockread(&ts, &host_tsc))
local_tsc_khz = 0; /* Fall back to old method */
put_cpu();
/*
* These values must be snapshotted within the seqcount loop.
* After that, it's just mathematics which can happen on any
* CPU at any time.
*/
hv_clock.tsc_timestamp = ka->master_cycle_now;
hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
} while (read_seqcount_retry(&ka->pvclock_sc, seq));
get_kvmclock(kvm, &data);
/*
* If the conditions were right, and obtaining the wallclock+TSC was
* successful, calculate the KVM clock at the corresponding time and
* subtract one from the other to get the guest's epoch in nanoseconds
* since 1970-01-01.
* If get_kvmclock() captured both wallclock and kvmclock from the
* same TSC reading, use them for a precise epoch calculation.
*/
if (local_tsc_khz) {
kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
&hv_clock.tsc_shift,
&hv_clock.tsc_to_system_mul);
return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
__pvclock_read_cycles(&hv_clock, host_tsc);
}
#endif
return ktime_get_real_ns() - get_kvmclock_ns(kvm);
if (data.flags & KVM_CLOCK_REALTIME)
return data.realtime - data.clock;
return ktime_get_real_ns() - data.clock;
}
/*
@ -2583,7 +2537,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
mark_tsc_unstable("KVM discovered backwards TSC");
if (kvm_check_tsc_unstable()) {
u64 offset = kvm_compute_l1_tsc_offset(vcpu,
u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(),
vcpu->arch.last_guest_tsc);
kvm_vcpu_write_tsc_offset(vcpu, offset);
if (!vcpu->arch.guest_tsc_protected)
@ -9444,6 +9398,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
return;
vcpu_load(vcpu);
kvm_synchronize_tsc(vcpu, NULL);
if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
kvm_update_masterclock(vcpu->kvm);
vcpu_put(vcpu);
/* poll control enabled by default */

3
arch/x86/kvm/x86.h
View File

@ -335,7 +335,8 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio);
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier);
u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier);
u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc);
u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc,
u64 target_tsc);
void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset);
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,