diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ab5b90c0d77c..4b070981016a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -870,7 +870,7 @@ struct kvm_vcpu_arch { gpa_t time; s8 pvclock_tsc_shift; u32 pvclock_tsc_mul; - unsigned int hw_tsc_khz; + u64 hw_tsc_hz; struct gfn_to_pfn_cache pv_time; /* set guest stopped flag in pvclock flags field */ bool pvclock_set_guest_stopped_request; @@ -1242,6 +1242,7 @@ struct kvm_arch { u64 last_tsc_write; u32 last_tsc_khz; u64 last_tsc_offset; + u64 last_tsc_scaling_ratio; u64 cur_tsc_nsec; u64 cur_tsc_write; u64 cur_tsc_offset; @@ -1257,6 +1258,9 @@ struct kvm_arch { u64 master_kernel_ns; u64 master_cycle_now; struct ratelimit_state kvmclock_update_rs; + u64 master_tsc_scaling_ratio; + s8 master_tsc_shift; + u32 master_tsc_mul; #ifdef CONFIG_KVM_HYPERV struct kvm_hv hyperv; diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 851f151efb35..38450e9fca9a 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index dd3bb04878ca..37d5bed3529f 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -1631,7 +1631,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (msr_info->host_initiated) { kvm_synchronize_tsc(vcpu, &data); } else if (!vcpu->arch.guest_tsc_protected) { - u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; + u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) - + vcpu->arch.l1_tsc_offset; adjust_tsc_offset_guest(vcpu, adj); vcpu->arch.ia32_tsc_adjust_msr += adj; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index fe0cba4731b2..103197238d24 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5617,7 +5617,8 @@ static __init int svm_hardware_setup(void) XFEATURE_MASK_BNDCSR); if (tsc_scaling) { - if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { + if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR) || + !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { tsc_scaling = false; } else { pr_info("TSC scaling supported\n"); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..54d93167e342 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2843,6 +2843,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, if (!cpu_has_sgx()) _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; + /* + * KVM doesn't re-derive the TSC scaling ratio when the host TSC + * frequency changes, so TSC scaling is only usable with a constant + * TSC. Clear the control here rather than in vmx_hardware_setup() so + * that the per-CPU configs recomputed by vmx_check_processor_compat() + * stay consistent with the golden vmcs_config. + */ + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) + _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING; + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) _cpu_based_3rd_exec_control = adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 40ebfc41ce72..dc3ed1a05fbd 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -35,6 +35,7 @@ #include "smm.h" #include +#include #include #include #include @@ -61,6 +62,7 @@ #include #include #include +#include #include #include @@ -925,19 +927,13 @@ static void update_pvclock_gtod(struct timekeeper *tk) write_seqcount_end(&vdata->seq); } +#endif static s64 get_kvmclock_base_ns(void) { /* Count up from boot time, but with the frequency of the raw clock. */ - return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot)); + return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); } -#else -static s64 get_kvmclock_base_ns(void) -{ - /* Master clock not used, so we can just use CLOCK_BOOTTIME. */ - return ktime_get_boottime_ns(); -} -#endif static uint32_t div_frac(uint32_t dividend, uint32_t divisor) { @@ -945,32 +941,57 @@ static uint32_t div_frac(uint32_t dividend, uint32_t divisor) return dividend; } -static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz, +static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz, s8 *pshift, u32 *pmultiplier) { - uint64_t scaled64; - int32_t shift = 0; - uint64_t tps64; - uint32_t tps32; + u64 scaled_hz_u64 = scaled_hz; + s32 shift = 0; + u64 base_hz_u64; + u32 base32; - tps64 = base_hz; - scaled64 = scaled_hz; - while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { - tps64 >>= 1; + /* + * This function calculates a fixed-point multiplier and shift such + * that: + * time_ns = (tsc_cycles << shift) * multiplier >> 32 + * + * Where tsc_cycles tick at base_hz, and time_ns should count at + * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion). + * + * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift + * to keep everything in range. + */ + + base_hz_u64 = base_hz; + + /* + * Start by shifting base_hz right until it fits in 32 bits, and + * is lower than double the target rate. This introduces a negative + * shift value which would result in pvclock_scale_delta() shifting + * the actual tick count right before performing the multiplication. + */ + while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) { + base_hz_u64 >>= 1; shift--; } - tps32 = (uint32_t)tps64; - while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { - if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) - scaled64 >>= 1; + /* Now the shifted base_hz fits in 32 bits. */ + base32 = (u32)base_hz_u64; + + /* + * Next, shift scaled_hz right until it fits in 32 bits, and ensure + * that the shifted base_hz is strictly larger (so that the result of the + * final division also fits in 32 bits). + */ + while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) { + if (scaled_hz_u64 >> 32 || base32 & BIT(31)) + scaled_hz_u64 >>= 1; else - tps32 <<= 1; + base32 <<= 1; shift++; } *pshift = shift; - *pmultiplier = div_frac(scaled64, tps32); + *pmultiplier = div_frac(scaled_hz_u64, base32); } #ifdef CONFIG_X86_64 @@ -1061,11 +1082,15 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) { - u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, - vcpu->arch.virtual_tsc_mult, - vcpu->arch.virtual_tsc_shift); - tsc += vcpu->arch.this_tsc_write; - return tsc; + s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec; + u64 tsc; + + /* Handle negative deltas gracefully (master clock ref may be earlier) */ + tsc = pvclock_scale_delta(abs(delta_ns), + vcpu->arch.virtual_tsc_mult, + vcpu->arch.virtual_tsc_shift); + + return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc); } #ifdef CONFIG_X86_64 @@ -1131,11 +1156,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio) return _tsc; } -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc) { u64 tsc; - tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); + tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); return target_tsc - tsc; } @@ -1254,6 +1280,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, kvm->arch.last_tsc_write = tsc; kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; kvm->arch.last_tsc_offset = offset; + kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; vcpu->arch.last_guest_tsc = tsc; @@ -1296,7 +1323,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) bool synchronizing = false; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); ns = get_kvmclock_base_ns(); elapsed = ns - kvm->arch.last_tsc_nsec; @@ -1345,7 +1372,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) } else { u64 delta = nsec_to_cycles(vcpu, elapsed); data += delta; - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); } matched = true; } @@ -1356,126 +1383,27 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) #ifdef CONFIG_X86_64 -static u64 read_tsc(void) +static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, + u64 *tsc_timestamp) { - u64 ret = (u64)rdtsc_ordered(); - u64 last = pvclock_gtod_data.clock.cycle_last; - - if (likely(ret >= last)) - return ret; - /* - * GCC likes to generate cmov here, but this branch is extremely - * predictable (it's just a function of time and the likely is - * very likely) and there's a data dependence, so force GCC - * to generate a branch instead. I don't barrier() because - * we don't actually need a barrier, and if this function - * ever gets inlined it will generate worse code. + * ktime_get_snapshot_id() cannot fail for standard clock IDs + * (only for invalid/aux clocks or during suspend, with a WARN). */ - asm volatile (""); - return last; -} + if (!snap->valid) + return false; -static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, - int *mode) -{ - u64 tsc_pg_val; - long v; - - switch (clock->vclock_mode) { - case VDSO_CLOCKMODE_HVCLOCK: - if (hv_read_tsc_page_tsc(hv_get_tsc_page(), - tsc_timestamp, &tsc_pg_val)) { - /* TSC page valid */ - *mode = VDSO_CLOCKMODE_HVCLOCK; - v = (tsc_pg_val - clock->cycle_last) & - clock->mask; - } else { - /* TSC page invalid */ - *mode = VDSO_CLOCKMODE_NONE; - } - break; - case VDSO_CLOCKMODE_TSC: - *mode = VDSO_CLOCKMODE_TSC; - *tsc_timestamp = read_tsc(); - v = (*tsc_timestamp - clock->cycle_last) & - clock->mask; - break; - default: - *mode = VDSO_CLOCKMODE_NONE; + if (snap->cs_id == CSID_X86_TSC) { + *tsc_timestamp = snap->cycles; + return true; } - if (*mode == VDSO_CLOCKMODE_NONE) - *tsc_timestamp = v = 0; + if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) { + *tsc_timestamp = snap->hw_cycles; + return true; + } - return v * clock->mult; -} - -/* - * As with get_kvmclock_base_ns(), this counts from boot time, at the - * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot). - */ -static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->raw_clock.base_cycles; - ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode); - ns >>= gtod->raw_clock.shift; - ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot)); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - -/* - * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with - * no boot time offset. - */ -static int do_monotonic(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - ns += ktime_to_ns(gtod->clock.offset); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - -static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ts->tv_sec = gtod->wall_time_sec; - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return mode; + return false; } /* @@ -1485,12 +1413,14 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) */ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT)); + return true; } /* @@ -1499,12 +1429,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) */ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_monotonic(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(snap.systime); + return true; } /* @@ -1517,11 +1449,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_REALTIME, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp)); + *ts = ktime_to_timespec64(snap.systime); + return true; } #endif @@ -1566,6 +1501,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, * */ +static unsigned long get_cpu_tsc_khz(void); + static void pvclock_update_vm_gtod_copy(struct kvm *kvm) { #ifdef CONFIG_X86_64 @@ -1589,9 +1526,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm) && !ka->backwards_tsc_observed && !ka->boot_vcpu_runs_old_kvmclock; - if (ka->use_master_clock) + if (ka->use_master_clock) { + u64 tsc_hz; + atomic_set(&kvm_guest_has_master_clock, 1); + /* + * Copy the scaling ratio and precompute the mul/shift for + * converting guest TSC to nanoseconds. These are used by + * get_kvmclock() to compute kvmclock from the host TSC + * without needing a vCPU reference. + */ + ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio; + tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (tsc_hz && kvm_caps.has_tsc_control) + tsc_hz = kvm_scale_tsc(tsc_hz, + ka->master_tsc_scaling_ratio); + if (tsc_hz) + kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, + &ka->master_tsc_shift, + &ka->master_tsc_mul); + else + ka->use_master_clock = false; + } + vclock_mode = pvclock_gtod_data.clock.vclock_mode; trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, vcpus_matched); @@ -1658,39 +1616,49 @@ static unsigned long get_cpu_tsc_khz(void) } /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ -static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) +static bool __get_kvmclock_master_clock(struct kvm *kvm, + struct kvm_clock_data *data) { +#ifdef CONFIG_X86_64 struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; + struct timespec64 ts; - /* both __this_cpu_read() and rdtsc() should be on the same cpu */ - get_cpu(); + if (!ka->use_master_clock) + return false; - data->flags = 0; - if (ka->use_master_clock && - (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) { -#ifdef CONFIG_X86_64 - struct timespec64 ts; + if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) + return false; - if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { - data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; - data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; - } else -#endif - data->host_tsc = rdtsc(); + data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; + data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC | + KVM_CLOCK_TSC_STABLE; - data->flags |= KVM_CLOCK_TSC_STABLE; - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); + hv_clock.tsc_timestamp = ka->master_cycle_now; + hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; + + /* + * Use the precomputed guest-TSC-based mul/shift so that the kvmclock + * value matches what the guest computes from its own TSC. + */ + hv_clock.tsc_shift = ka->master_tsc_shift; + hv_clock.tsc_to_system_mul = ka->master_tsc_mul; + + if (kvm_caps.has_tsc_control) { + u64 tsc_delta = data->host_tsc - ka->master_cycle_now; + + tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio); + data->clock = hv_clock.system_time + + pvclock_scale_delta(tsc_delta, + hv_clock.tsc_to_system_mul, + hv_clock.tsc_shift); } else { - data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; + data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); } - - put_cpu(); + return true; +#else + return false; +#endif } static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) @@ -1699,8 +1667,11 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) unsigned seq; do { + data->flags = 0; + seq = read_seqcount_begin(&ka->pvclock_sc); - __get_kvmclock(kvm, data); + if (!__get_kvmclock_master_clock(kvm, data)) + data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; } while (read_seqcount_retry(&ka->pvclock_sc, seq)); } @@ -1762,36 +1733,45 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { + u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns; + struct kvm_arch *ka __maybe_unused = &v->kvm->arch; struct pvclock_vcpu_time_info hv_clock = {}; - unsigned long flags, tgt_tsc_khz; - unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; - struct kvm_arch *ka = &v->kvm->arch; s64 kernel_ns; - u64 tsc_timestamp, host_tsc; - bool use_master_clock; - - kernel_ns = 0; - host_tsc = 0; /* * If the host uses TSC clock, then passthrough TSC as stable * to the guest. */ +#ifdef CONFIG_X86_64 + bool use_master_clock; + unsigned int seq; + do { seq = read_seqcount_begin(&ka->pvclock_sc); use_master_clock = ka->use_master_clock; - if (use_master_clock) { - host_tsc = ka->master_cycle_now; - kernel_ns = ka->master_kernel_ns; - } - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + if (!use_master_clock) + continue; - /* Keep irq disabled to prevent changes to the clock */ - local_irq_save(flags); - tgt_tsc_khz = get_cpu_tsc_khz(); - if (unlikely(tgt_tsc_khz == 0)) { - local_irq_restore(flags); + if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) { + use_master_clock = false; + continue; + } + + master_tsc = ka->master_cycle_now; + master_ns = ka->master_kernel_ns; + } while (read_seqcount_retry(&ka->pvclock_sc, seq)); +#else + const bool use_master_clock = false; +#endif + /* + * Ensure reading the TSC+frequency pair is done on the same CPU. When + * NOT using the master clock, the TSC frequency may vary between CPUs. + */ + preempt_disable(); + tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (unlikely(tgt_tsc_hz == 0)) { + preempt_enable(); kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); return 1; } @@ -1820,28 +1800,43 @@ int kvm_guest_time_update(struct kvm_vcpu *v) } } - local_irq_restore(flags); + /* + * Refresh L1's last "observed" TSC to match the PV clock's timestamp, + * e.g. so that the guest can't see a TSC that's behind the reference. + */ + vcpu->last_guest_tsc = tsc_timestamp; + + preempt_enable(); /* With all the info we got, fill in the values */ if (kvm_caps.has_tsc_control) { - tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz, + tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz, v->arch.l1_tsc_scaling_ratio); - tgt_tsc_khz = tgt_tsc_khz ? : 1; + tgt_tsc_hz = tgt_tsc_hz ? : 1; } - if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { - kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL, + if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) { + kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz, &vcpu->pvclock_tsc_shift, &vcpu->pvclock_tsc_mul); - vcpu->hw_tsc_khz = tgt_tsc_khz; + vcpu->hw_tsc_hz = tgt_tsc_hz; } hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; - hv_clock.tsc_timestamp = tsc_timestamp; - hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; - vcpu->last_guest_tsc = tsc_timestamp; + /* + * If the master clock is NOT in use, the reference time placed in the + * hv_clock is "now". If master clock is in use, the reference time is + * the master clock's snapshot from some time in the past, not "now". + */ + if (use_master_clock) { + hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc); + hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset; + } else { + hv_clock.tsc_timestamp = tsc_timestamp; + hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; + } /* If the host uses TSC clocksource, then it is stable */ hv_clock.flags = 0; @@ -1903,63 +1898,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v) * wallclock and kvmclock times, and subtracting one from the other. * * Fall back to using their values at slightly different moments by - * calling ktime_get_real_ns() and get_kvmclock_ns() separately. + * calling ktime_get_real_ns() and get_kvmclock() separately. */ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) { -#ifdef CONFIG_X86_64 - struct pvclock_vcpu_time_info hv_clock; - struct kvm_arch *ka = &kvm->arch; - unsigned long seq, local_tsc_khz; - struct timespec64 ts; - uint64_t host_tsc; + struct kvm_clock_data data; - do { - seq = read_seqcount_begin(&ka->pvclock_sc); - - local_tsc_khz = 0; - if (!ka->use_master_clock) - break; - - /* - * The TSC read and the call to get_cpu_tsc_khz() must happen - * on the same CPU. - */ - get_cpu(); - - local_tsc_khz = get_cpu_tsc_khz(); - - if (local_tsc_khz && - !kvm_get_walltime_and_clockread(&ts, &host_tsc)) - local_tsc_khz = 0; /* Fall back to old method */ - - put_cpu(); - - /* - * These values must be snapshotted within the seqcount loop. - * After that, it's just mathematics which can happen on any - * CPU at any time. - */ - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + get_kvmclock(kvm, &data); /* - * If the conditions were right, and obtaining the wallclock+TSC was - * successful, calculate the KVM clock at the corresponding time and - * subtract one from the other to get the guest's epoch in nanoseconds - * since 1970-01-01. + * If get_kvmclock() captured both wallclock and kvmclock from the + * same TSC reading, use them for a precise epoch calculation. */ - if (local_tsc_khz) { - kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec - - __pvclock_read_cycles(&hv_clock, host_tsc); - } -#endif - return ktime_get_real_ns() - get_kvmclock_ns(kvm); + if (data.flags & KVM_CLOCK_REALTIME) + return data.realtime - data.clock; + + return ktime_get_real_ns() - data.clock; } /* @@ -2583,7 +2537,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) mark_tsc_unstable("KVM discovered backwards TSC"); if (kvm_check_tsc_unstable()) { - u64 offset = kvm_compute_l1_tsc_offset(vcpu, + u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), vcpu->arch.last_guest_tsc); kvm_vcpu_write_tsc_offset(vcpu, offset); if (!vcpu->arch.guest_tsc_protected) @@ -9444,6 +9398,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) return; vcpu_load(vcpu); kvm_synchronize_tsc(vcpu, NULL); + if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) + kvm_update_masterclock(vcpu->kvm); vcpu_put(vcpu); /* poll control enabled by default */ diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 0f5919b092e4..1d9a66048b01 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -335,7 +335,8 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio); u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc); +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc); void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,