KVM selftests changes and a few KVM x86 fixes for 7.4

- Fix emulation of VMPTRST and VMPTRLD when eVMCS is in-use to match real
    Hyper-V's behavior.
 
  - Saturate L2's TSC frequency if stacking L1 and L2 scaling multipliers
    exceeds what hardware supports, as attempting to run L2 with an illegal
    multiplier will either cause VM-Entry to fail (VMX) or trigger a #GP when
    writing MSR_AMD64_TSC_RATIO (SVM).  The #GP is thankfully benign in the
    grand scheme as even "unsafe" WRMSRs have exception fixup.
 
  - Rework KVM selftest's handling of memory regions so that PPC can force its
    page table allocations to be naturally aligned.
 
  - Clean up selftest's VMX instruction helpers, and drop an absurd number of
    completely useless return values, e.g. functions that unconditionally
    return %true and then have every caller assert success.
 
  - Add selftests support for Zhaoxin CPUs.
 
  - Overhaul the TSC scaling test to add a regression test for the saturation
    behavior, and to increase its coverage in general.
 
  - Fix selftests usage various NUMA syscalls to account for an undocumented,
    but known since 2007, off-by-one kernel bug.  Add helpers to try and avoid
    re-introducing the same flaws in the future.
 
  - Automatically run xAPIC IPI migration test (between NUMA nodes) when
    possible, as the entire point of the test is verify KVM's handling of page
    migration of the APIC backing page.
 
  - Misc cleanups.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmrRp9gACgkQOlYIJqCj
 N/2CCxAAg0sCpc8pkmZjmUQjyjFN1WSefjneOz5qrXNEpTLij3ZwBds1Ns/KLykQ
 QJT09lsYI4Od7OmUpF9LwHuxHXyG09i7FNk5CCP+FM2YmD+X5+o9mqhj9+/V6nBT
 opvDeXOJkrfqN+33Y+dQl/APjnVT/ml9i0C3vkmfH8inWBbWCYza/VCrFXQjFl1p
 SsdhIEP1lnQyYD2N7Im2rBRuGjPBlaKSea3ypTmJyrRL8QJ4aNOj9yZKgHUYHq+W
 1vPla1vocwdKd9thPdU0+C4aY4eJ895bkMfHwTPgUyGxdfxrYng1KoOI8k5aGHZi
 KNR3rf7lZQ9ZSmtNtLpW2GLD9mUzzhZJAjFGEMWli/yOB4R4stpT0of/2eZwoEgJ
 fkd834ZVjn+8F0VZaKcyLJVQMxNh/PQQAL2Jo+6dUP0KNbJEDPrKf32uFQERebIg
 sO/DoKXWKG3721PynMMz3lfNMayZfOeuqdXMpuT3BPQj1EdU9qnO25/02+3CejMY
 Hb9QbMijnsX1X3SiJs2m0XyQoGHTwTkhjIDh6fbKhrUTUQVX4qC3V36Jwi22j1pC
 vLWQ5A+tPwV6dZAZDhpD5sreCg9aocxhYT77fzV27kIMpqtY0PM7GWhJqiveG25m
 xPAOoRVoEdf1eOzri1Oit6sHaPTNYey4HG0pfV0CJEXJf4F7UFA=
 =mmOS
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-selftests-7.4' of https://github.com/kvm-x86/linux into HEAD

KVM selftests changes and a few KVM x86 fixes for 7.4

 - Fix emulation of VMPTRST and VMPTRLD when eVMCS is in-use to match real
   Hyper-V's behavior.

 - Saturate L2's TSC frequency if stacking L1 and L2 scaling multipliers
   exceeds what hardware supports, as attempting to run L2 with an illegal
   multiplier will either cause VM-Entry to fail (VMX) or trigger a #GP when
   writing MSR_AMD64_TSC_RATIO (SVM).  The #GP is thankfully benign in the
   grand scheme as even "unsafe" WRMSRs have exception fixup.

 - Rework KVM selftest's handling of memory regions so that PPC can force its
   page table allocations to be naturally aligned.

 - Clean up selftest's VMX instruction helpers, and drop an absurd number of
   completely useless return values, e.g. functions that unconditionally
   return %true and then have every caller assert success.

 - Add selftests support for Zhaoxin CPUs.

 - Overhaul the TSC scaling test to add a regression test for the saturation
   behavior, and to increase its coverage in general.

 - Fix selftests usage various NUMA syscalls to account for an undocumented,
   but known since 2007, off-by-one kernel bug.  Add helpers to try and avoid
   re-introducing the same flaws in the future.

 - Automatically run xAPIC IPI migration test (between NUMA nodes) when
   possible, as the entire point of the test is verify KVM's handling of page
   migration of the APIC backing page.

 - Misc cleanups.

arch/x86/kvm/svm/svm.c

@@ -533,6 +533,15 @@ static int svm_check_processor_compat(void)
 
 static void __svm_write_tsc_multiplier(u64 multiplier)
 {
+	/*
+	 * Fallback to the default ratio if KVM is buggy and tries to program
+	 * an unsupported scaling ratio, e.g. so that the guest has a chance of
+	 * surviving, so that the cache isn't stale/corrupted, and so that KVM
+	 * doesn't leak state across VMs.
+	 */
+	if (WARN_ON_ONCE(multiplier & SVM_TSC_RATIO_RSVD))
+		multiplier = SVM_TSC_RATIO_DEFAULT;
+
 	if (multiplier == __this_cpu_read(current_tsc_ratio))
 		return;
 

arch/x86/kvm/vmx/nested.c

@@ -5898,6 +5898,12 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
+	/* Forbid normal VMPTRLD if Enlightened version was used */
+	if (nested_vmx_is_evmptr12_valid(vmx)) {
+		kvm_queue_exception(vcpu, UD_VECTOR);
+		return 1;
+	}
+
 	if (nested_vmx_get_vmptr(vcpu, &vmptr, &r))
 		return r;
 
@@ -5907,10 +5913,6 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 	if (vmptr == vmx->nested.vmxon_ptr)
 		return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
 
-	/* Forbid normal VMPTRLD if Enlightened version was used */
-	if (nested_vmx_is_evmptr12_valid(vmx))
-		return 1;
-
 	if (vmx->nested.current_vmptr != vmptr) {
 		struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache;
 		struct vmcs_hdr hdr;
@@ -5963,7 +5965,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
 {
 	unsigned long exit_qual = vmx_get_exit_qual(vcpu);
 	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
-	gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
+	gpa_t current_vmptr;
 	struct x86_exception e;
 	gva_t gva;
 	int r;
@@ -5971,8 +5973,16 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
 	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
-	if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu))))
+	/*
-		return 1;
+	 * Hyper-V TLFS does not specify the behavior of VMPTRST when eVMCS is used
+	 * but genuine Hyper-V seems to be returning eVMCS GPA.
+	 */
+#ifdef CONFIG_KVM_HYPERV
+	if (nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))
+		current_vmptr = to_vmx(vcpu)->nested.hv_evmcs_vmptr;
+	else
+#endif
+		current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
 
 	if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
 				true, sizeof(gpa_t), &gva))

arch/x86/kvm/x86.c

@@ -1200,11 +1200,58 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset);
 
 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
 {
-	if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
+	u8 frac_bits = kvm_caps.tsc_scaling_ratio_frac_bits;
-		return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
+	u64 nested_multiplier;
-				       kvm_caps.tsc_scaling_ratio_frac_bits);
 
-	return l1_multiplier;
+	if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
+		return l1_multiplier;
+
+	/*
+	 * The shift is fixed on both AMD and Intel, and operates on a 64-bit
+	 * value.  I.e. a shift greater than 63 is completely nonsensical.
+	 */
+	if (WARN_ON_ONCE(frac_bits > 63))
+		return l1_multiplier;
+
+	/*
+	 * If the resulting multiplier can't be programmed into hardware, run
+	 * L2 at the minimum/maximum frequency supported by hardware, i.e.
+	 * saturate L2's frequency on both sides.  Because L2's frequency needs
+	 * to be distilled down to a single multiplier to get from:
+	 *
+	 *     L2 = (((L0 * L1_mult) >> frac) * L2_mult) >> frac)
+	 *
+	 * to:
+	 *
+	 *     L2 = (L0 * mult) >> frac
+	 *
+	 * very small/large L1 and L2 multipliers can underflow/overflow the
+	 * minimum/maximum multiplier supported by hardware when combined into
+	 * a single value.
+	 *
+	 * Manually check for the case where the result would overflow a u64,
+	 * i.e. if the multiplier would be silently truncated before the "too
+	 * large" check.  Avoid doing the multiply twice in the common case
+	 * where the compiler natively supports 128-bit values.
+	 */
+#ifdef CONFIG_ARCH_SUPPORTS_INT128
+	unsigned __int128 m = (unsigned __int128)l1_multiplier * l2_multiplier;
+
+	if (m >> (64 + frac_bits))
+		return kvm_caps.max_tsc_scaling_ratio;
+
+	nested_multiplier = m >> frac_bits;
+#else
+	if (mul_u64_u64_shr(l1_multiplier, l2_multiplier, 64 + frac_bits))
+		return kvm_caps.max_tsc_scaling_ratio;
+
+	nested_multiplier = mul_u64_u64_shr(l1_multiplier, l2_multiplier, frac_bits);
+#endif
+	if (nested_multiplier > kvm_caps.max_tsc_scaling_ratio)
+		return kvm_caps.max_tsc_scaling_ratio;
+
+	/* The minimum multiplier is '1' on both AMD and Intel. */
+	return nested_multiplier ?: 1;
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier);
 
@@ -3681,7 +3728,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		user_tsc_khz = (u32)arg;
 
 		if (kvm_caps.has_tsc_control &&
-		    user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
+		    user_tsc_khz > kvm_caps.max_guest_tsc_khz)
 			goto out;
 
 		if (user_tsc_khz == 0)
@@ -4648,7 +4695,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 		user_tsc_khz = (u32)arg;
 
 		if (kvm_caps.has_tsc_control &&
-		    user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
+		    user_tsc_khz > kvm_caps.max_guest_tsc_khz)
 			goto out;
 
 		if (user_tsc_khz == 0)
@@ -7144,7 +7191,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)
 	if (kvm_caps.has_tsc_control) {
 		/*
 		 * Make sure the user can only configure tsc_khz values that
-		 * fit into a signed integer.
+		 * fit into a signed integer, otherwise KVM_GET_TSC_KHZ would
+		 * return a negative value and confuse userspace.
 		 * A min value is not calculated because it will always
 		 * be 1 on all machines.
 		 */

tools/testing/selftests/kvm/Makefile.kvm

@@ -6,7 +6,6 @@ all:
 LIBKVM += lib/assert.c
 LIBKVM += lib/elf.c
 LIBKVM += lib/guest_modes.c
-LIBKVM += lib/io.c
 LIBKVM += lib/kvm_util.c
 LIBKVM += lib/lru_gen_util.c
 LIBKVM += lib/memstress.c

tools/testing/selftests/kvm/arm64/hypercalls.c

@@ -187,7 +187,7 @@ struct st_time {
 
 static void steal_time_init(struct kvm_vcpu *vcpu)
 {
-	u64 st_ipa = (ulong)ST_GPA_BASE;
+	u64 st_ipa = (unsigned long)ST_GPA_BASE;
 	unsigned int gpages;
 
 	gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE);

tools/testing/selftests/kvm/arm64/page_fault_test.c

@@ -597,18 +597,16 @@ static void setup_memslots(struct kvm_vm *vm, struct test_params *p)
 
 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0,
 				    CODE_AND_DATA_MEMSLOT, code_npages, 0);
-	vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT;
+	____vm_override_mem_region(vm, MEM_REGION_CODE, CODE_AND_DATA_MEMSLOT);
-	vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT;
+	____vm_override_mem_region(vm, MEM_REGION_DATA, CODE_AND_DATA_MEMSLOT);
 
-	vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size,
+	__vm_override_mem_region(vm, MEM_REGION_PT, p->src_type, data_gpa - pt_size,
-				    PAGE_TABLE_MEMSLOT, pt_size / guest_page_size,
+				 PAGE_TABLE_MEMSLOT, pt_size / guest_page_size,
-				    p->test_desc->pt_memslot_flags);
+				 p->test_desc->pt_memslot_flags);
-	vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT;
 
-	vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT,
+	__vm_override_mem_region(vm, MEM_REGION_TEST_DATA, p->src_type, data_gpa,
-				    data_size / guest_page_size,
+				 TEST_DATA_MEMSLOT, data_size / guest_page_size,
-				    p->test_desc->data_memslot_flags);
+				 p->test_desc->data_memslot_flags);
-	vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT;
 }
 
 static void setup_ucall(struct kvm_vm *vm)

tools/testing/selftests/kvm/arm64/vgic_its_save.c

@@ -279,15 +279,15 @@ static void setup_memslot(size_t sz)
 	size_t pages = sz / vm->page_size;
 
 	gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz;
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base,
+	vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS,
-				    TEST_MEMSLOT_INDEX, pages, 0);
+			       gpa_base, TEST_MEMSLOT_INDEX, pages);
 }
 
 static gpa_t alloc_64k(size_t nr)
 {
 	size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K);
-	gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, gpa_base,
+	gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k,
-				       TEST_MEMSLOT_INDEX);
+				       MEM_REGION_TEST_EXTRA);
 
 	TEST_ASSERT(IS_ALIGNED(gpa, SZ_64K),
 		    "Allocation at 0x%lx is not 64K aligned, GITS_BASER cannot address it",

tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c

@@ -23,7 +23,6 @@
 #define GIC_LPI_OFFSET	8192
 
 static size_t nr_iterations = 1000;
-static gpa_t gpa_base;
 
 static struct kvm_vm *vm;
 static struct kvm_vcpu **vcpus;
@@ -143,6 +142,7 @@ static void guest_code(size_t nr_lpis)
 
 static void setup_memslot(void)
 {
+	gpa_t gpa_base;
 	size_t pages;
 	size_t sz;
 
@@ -164,8 +164,8 @@ static void setup_memslot(void)
 
 	pages = sz / vm->page_size;
 	gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz;
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base,
+	vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS,
-				    TEST_MEMSLOT_INDEX, pages, 0);
+			       gpa_base, TEST_MEMSLOT_INDEX, pages);
 }
 
 #define LPI_PROP_DEFAULT_PRIO	0xa0
@@ -191,28 +191,25 @@ static void setup_test_data(void)
 	gpa_t cmdq_base;
 
 	test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k,
-						    gpa_base,
+						    MEM_REGION_TEST_EXTRA);
-						    TEST_MEMSLOT_INDEX);
 
 	test_data.collection_table = vm_phy_pages_alloc(vm, pages_per_64k,
-							gpa_base,
+							MEM_REGION_TEST_EXTRA);
-							TEST_MEMSLOT_INDEX);
 
-	cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base,
+	cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, MEM_REGION_TEST_EXTRA);
-				       TEST_MEMSLOT_INDEX);
 	virt_map(vm, cmdq_base, cmdq_base, pages_per_64k);
 	test_data.cmdq_base = cmdq_base;
 	test_data.cmdq_base_va = (void *)cmdq_base;
 
 	test_data.itt_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_devices,
-						  gpa_base, TEST_MEMSLOT_INDEX);
+						  MEM_REGION_TEST_EXTRA);
 
 	test_data.lpi_prop_table = vm_phy_pages_alloc(vm, pages_per_64k,
-						      gpa_base, TEST_MEMSLOT_INDEX);
+						      MEM_REGION_TEST_EXTRA);
 	configure_lpis();
 
 	test_data.lpi_pend_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_cpus,
-						       gpa_base, TEST_MEMSLOT_INDEX);
+						       MEM_REGION_TEST_EXTRA);
 
 	sync_global_to_guest(vm, test_data);
 }

tools/testing/selftests/kvm/demand_paging_test.c

@@ -297,8 +297,7 @@ int main(int argc, char *argv[])
 			p.single_uffd = true;
 			break;
 		case 'd':
-			p.uffd_delay = strtoul(optarg, NULL, 0);
+			p.uffd_delay = atoi_non_negative("UFFD delay", optarg);
-			TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported.");
 			break;
 		case 'b':
 			guest_percpu_mem_size = parse_size(optarg);

tools/testing/selftests/kvm/guest_memfd_test.c

@@ -80,7 +80,7 @@ static void test_mbind(int fd, size_t total_size)
 {
 	const unsigned long nodemask_0 = 1; /* nid: 0 */
 	unsigned long nodemask = 0;
-	unsigned long maxnode = BITS_PER_TYPE(nodemask);
+	unsigned long maxnode = MAXNODE_FOR_MASK(nodemask);
 	int policy;
 	char *mem;
 	int ret;

tools/testing/selftests/kvm/include/kvm_util.h

@@ -33,6 +33,8 @@
 
 #define NSEC_PER_SEC 1000000000L
 
+#define KVM_INVALID_MEMSLOT UINT_MAX
+
 struct userspace_mem_region {
 	struct kvm_userspace_memory_region2 region;
 	struct sparsebit *unused_phy_pages;
@@ -80,11 +82,45 @@ struct userspace_mem_regions {
 	DECLARE_HASHTABLE(slot_hash, 9);
 };
 
+/*
+ * Memory region types are passed to various page allocators to communicate
+ * various properties and metadata related to the allocation.  Note, the
+ * descriptions below described the primary usage of each type.  Individual
+ * tests may allocate memory for other purposes.
+ *
+ * By default, all regions except TEST_EXTRA are mapped to memslot '0'.  The
+ * TEST_EXTRA region is left unmapped as it's intended to be used only for test
+ * specific allocations, i.e. should never be used by core/infrastructure code.
+ * Tests can override the memslot for any or all types, e.g. so that all test
+ * data is allocated from a curated memslot.
+ */
 enum kvm_mem_region_type {
+	/*
+	 * The CODE region is used by lib/elf when loading the test's code into
+	 * guest memory.
+	 */
 	MEM_REGION_CODE,
+	/*
+	 * The DATA region is used to allocate core data structures, e.g. vCPU
+	 * stacks, VM exception tables, x86's TSS, etc.
+	 */
 	MEM_REGION_DATA,
+	/*
+	 * The PT region, a.k.a. Page Table region, is used to allocate page
+	 * table pages.
+	 */
 	MEM_REGION_PT,
+	/*
+	 * The TEST_DATA region is used for allocating test data that is either
+	 * test specific, and/or isn't considered a "core" data structure.
+	 */
 	MEM_REGION_TEST_DATA,
+	/*
+	 * The TEST_EXTRA region is for special snowflakes, where a test wants
+	 * to create and use a one-off memslot, without impacting "normal" test
+	 * data allocations.
+	 */
+	MEM_REGION_TEST_EXTRA,
 	NR_MEM_REGIONS,
 };
 
@@ -127,11 +163,6 @@ struct kvm_vm {
 
 	struct kvm_binary_stats stats;
 
-	/*
-	 * KVM region slots. These are the default memslots used by page
-	 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
-	 * memslot.
-	 */
 	u32 memslots[NR_MEM_REGIONS];
 };
 
@@ -169,7 +200,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot);
 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
 							     enum kvm_mem_region_type type)
 {
-	assert(type < NR_MEM_REGIONS);
+	TEST_ASSERT(type < NR_MEM_REGIONS,
+		    "Invalid memory region type '%u'", type);
 	return memslot2region(vm, vm->memslots[type]);
 }
 
@@ -805,6 +837,36 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
 		gpa_t gpa, u32 slot, u64 npages, u32 flags,
 		int gmem_fd, u64 gmem_offset, u64 gmem_flags);
 
+
+static inline void ____vm_override_mem_region(struct kvm_vm *vm,
+					      enum kvm_mem_region_type type,
+					      u32 slot)
+{
+	TEST_ASSERT(vm->memslots[type] == KVM_INVALID_MEMSLOT,
+		    "Memory region type '%u' was already overridden with slot=%u",
+		    type, vm->memslots[type]);
+
+	vm->memslots[type] = slot;
+}
+
+static inline void __vm_override_mem_region(struct kvm_vm *vm,
+					    enum kvm_mem_region_type type,
+					    enum vm_mem_backing_src_type src_type,
+					    gpa_t gpa, u32 slot, u64 npages,
+					    u32 flags)
+{
+	____vm_override_mem_region(vm, type, slot);
+	vm_userspace_mem_region_add(vm, src_type, gpa, slot, npages, flags);
+}
+
+static inline void vm_override_mem_region(struct kvm_vm *vm,
+					  enum kvm_mem_region_type type,
+					  enum vm_mem_backing_src_type src_type,
+					  gpa_t gpa, u32 slot, u64 npages)
+{
+	__vm_override_mem_region(vm, type, src_type, gpa, slot, npages, 0);
+}
+
 #ifndef vm_arch_has_protected_memory
 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
 {
@@ -1098,23 +1160,41 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 
 const char *exit_reason_str(unsigned int exit_reason);
 
-gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot);
+bool kvm_arch_needs_naturally_aligned_page_tables(void);
-gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, gpa_t min_gpa,
-			   u32 memslot, bool protected);
-gpa_t vm_alloc_page_table(struct kvm_vm *vm);
 
-static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
+gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa,
-				       gpa_t min_gpa, u32 memslot)
+			     u32 memslot, bool protected, bool naturally_aligned);
+gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages,
+			   enum kvm_mem_region_type type, bool protected);
+
+static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages,
+				       enum kvm_mem_region_type type)
 {
 	/*
 	 * By default, allocate memory as protected for VMs that support
 	 * protected memory, as the majority of memory for such VMs is
 	 * protected, i.e. using shared memory is effectively opt-in.
 	 */
-	return __vm_phy_pages_alloc(vm, num, min_gpa, memslot,
+	return __vm_phy_pages_alloc(vm, nr_pages, type,
 				    vm_arch_has_protected_memory(vm));
 }
 
+static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm,
+				      enum kvm_mem_region_type type)
+{
+	return vm_phy_pages_alloc(vm, 1, type);
+}
+
+static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages)
+{
+	return vm_phy_pages_alloc(vm, nr_pages, MEM_REGION_PT);
+}
+
+static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm)
+{
+	return vm_alloc_page_table_pages(vm, 1);
+}
+
 /*
  * ____vm_create() does KVM_CREATE_VM and little else.  __vm_create() also
  * loads the test binary into guest memory and creates an IRQ chip (x86 only).

tools/testing/selftests/kvm/include/numaif.h

@@ -6,6 +6,7 @@
 
 #include <dirent.h>
 
+#include <linux/bitmap.h>
 #include <linux/mempolicy.h>
 
 #include "kvm_syscalls.h"
@@ -30,6 +31,48 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode,
 		   const unsigned long *, nodemask, unsigned long, maxnode,
 		   unsigned int, flags);
 
+/*
+ * Calculate the @maxnode param for the above syscalls given the mask that will
+ * be passed to the kernel, to account for a longstanding off-by-one bug in the
+ * kernel that isn't properly documented in the manpages.  The manpages say
+ * that @maxnode is "the maximum node ID plus one", but the kernel's actual
+ * behavior is "the number of bits in the mask plus one", i.e. "the maximum
+ * node ID plus two".
+ */
+#define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1)
+
+static inline int kvm_get_numa_memory_nodes(unsigned long *nodemask)
+{
+	int r;
+
+	*nodemask = 0;
+
+	r = get_mempolicy(NULL, nodemask, MAXNODE_FOR_MASK(*nodemask), 0,
+			  MPOL_F_MEMS_ALLOWED);
+	TEST_ASSERT(!r || errno == ENOSYS || errno == EPERM,
+		    "Unexpected get_mempolicy() failure");
+	return __builtin_popcountl(*nodemask);
+}
+
+/*
+ * Return the node ID of the next NUMA node in the mask, starting at @from+1.
+ * Guarantees a node is found, and that the found node is not @from.  Pass -1
+ * to find the first node in the mask.
+ */
+static inline int kvm_get_next_numa_node(unsigned long nodemask, int from)
+{
+	const unsigned long nr_bits = BITS_PER_TYPE(nodemask);
+	int to;
+
+	to = find_next_bit(&nodemask, nr_bits, from + 1);
+	if (to == nr_bits)
+		to = find_next_bit(&nodemask, nr_bits, 0);
+
+	TEST_ASSERT(to != nr_bits && to != from,
+		    "Unabled to find second NUMA node (from = %d, to = %d)", from, to);
+	return to;
+}
+
 static inline int get_max_numa_node(void)
 {
 	struct dirent *de;

tools/testing/selftests/kvm/include/test_util.h

@@ -49,10 +49,6 @@ do {								\
 
 #define TEST_REQUIRE(f) __TEST_REQUIRE(f, "Requirement not met: %s", #f)
 
-ssize_t test_write(int fd, const void *buf, size_t count);
-ssize_t test_read(int fd, void *buf, size_t count);
-int test_seq_read(const char *path, char **bufp, size_t *sizep);
-
 void __printf(5, 6) test_assert(bool exp, const char *exp_str,
 				const char *file, unsigned int line,
 				const char *fmt, ...);
@@ -243,7 +239,6 @@ static inline u32 atoi_non_negative(const char *name, const char *num_str)
 }
 
 int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args);
-__printf(3, 4) int guest_snprintf(char *buf, int n, const char *fmt, ...);
 
 char *strdup_printf(const char *fmt, ...) __attribute__((format(printf, 1, 2), nonnull(1)));
 

tools/testing/selftests/kvm/include/x86/evmcs.h

@@ -245,32 +245,18 @@ static inline void evmcs_enable(void)
 	enable_evmcs = true;
 }
 
-static inline int evmcs_vmptrld(u64 vmcs_pa, void *vmcs)
+static inline void evmcs_vmptrld(u64 vmcs_pa, void *vmcs)
 {
 	current_vp_assist->current_nested_vmcs = vmcs_pa;
 	current_vp_assist->enlighten_vmentry = 1;
 
 	current_evmcs = vmcs;
-
-	return 0;
 }
 
-static inline bool load_evmcs(struct hyperv_test_pages *hv)
+static inline void load_evmcs(struct hyperv_test_pages *hv)
 {
-	if (evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs))
+	evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs);
-		return false;
-
 	current_evmcs->revision_id = EVMCS_VERSION;
-
-	return true;
-}
-
-static inline int evmcs_vmptrst(u64 *value)
-{
-	*value = current_vp_assist->current_nested_vmcs &
-		~HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;
-
-	return 0;
 }
 
 static inline int evmcs_vmread(u64 encoding, u64 *value)
@@ -1201,62 +1187,4 @@ static inline int evmcs_vmwrite(u64 encoding, u64 value)
 	return 0;
 }
 
-static inline int evmcs_vmlaunch(void)
-{
-	int ret;
-
-	current_evmcs->hv_clean_fields = 0;
-
-	__asm__ __volatile__("push $0;"
-			     "mov %%rsp, (%[host_rsp]);"
-			     "lea 1f(%%rip), %%rax;"
-			     "mov %%rax, (%[host_rip]);"
-			     VMX_SWITCH_GPRS_ASM
-			     "vmlaunch;"
-			     "incq (%%rsp);"
-			     "1: ;"
-			     VMX_SWITCH_GPRS_ASM
-			     "pop %%rax;"
-			     : [ret]"=&a"(ret)
-			     : [host_rsp]"r"
-			       ((u64)&current_evmcs->host_rsp),
-			       [host_rip]"r"
-			       ((u64)&current_evmcs->host_rip),
-			       GUEST_REGS_OFFSETS
-			     : "memory", "cc");
-	return ret;
-}
-
-/*
- * No guest state (e.g. GPRs) is established by this vmresume.
- */
-static inline int evmcs_vmresume(void)
-{
-	int ret;
-
-	/* HOST_RIP */
-	current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1;
-	/* HOST_RSP */
-	current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER;
-
-	__asm__ __volatile__("push $0;"
-			     "mov %%rsp, (%[host_rsp]);"
-			     "lea 1f(%%rip), %%rax;"
-			     "mov %%rax, (%[host_rip]);"
-			     VMX_SWITCH_GPRS_ASM
-			     "vmresume;"
-			     "incq (%%rsp);"
-			     "1: ;"
-			     VMX_SWITCH_GPRS_ASM
-			     "pop %%rax;"
-			     : [ret]"=&a"(ret)
-			     : [host_rsp]"r"
-			       ((u64)&current_evmcs->host_rsp),
-			       [host_rip]"r"
-			       ((u64)&current_evmcs->host_rip),
-			       GUEST_REGS_OFFSETS
-			     : "memory", "cc");
-	return ret;
-}
-
 #endif /* !SELFTEST_KVM_EVMCS_H */

tools/testing/selftests/kvm/include/x86/processor.h

@@ -23,6 +23,8 @@ extern bool host_cpu_is_intel;
 extern bool host_cpu_is_amd;
 extern bool host_cpu_is_hygon;
 extern bool host_cpu_is_amd_compatible;
+extern bool host_cpu_is_zhaoxin;
+extern bool host_cpu_is_intel_compatible;
 extern u64 guest_tsc_khz;
 extern struct kvm_mmu guest_mmu;
 
@@ -783,6 +785,12 @@ static inline bool this_cpu_is_hygon(void)
 	return this_cpu_vendor_string_is("HygonGenuine");
 }
 
+static inline bool this_cpu_is_zhaoxin(void)
+{
+	return this_cpu_vendor_string_is("CentaurHauls") ||
+	       this_cpu_vendor_string_is("  Shanghai  ");
+}
+
 static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi)
 {
 	u32 gprs[4];
@@ -1173,7 +1181,6 @@ static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu)
 void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,
 			     struct kvm_x86_cpu_property property,
 			     u32 value);
-void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, u8 maxphyaddr);
 
 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function);
 
@@ -1319,7 +1326,7 @@ gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages);
 	"lea 1f(%%rip), %%r10\n\t"				\
 	"lea 2f(%%rip), %%r11\n\t"				\
 	fep "1: " insn "\n\t"					\
-	"xor %%r9, %%r9\n\t"					\
+	"mov $0, %%r9d\n\t"					\
 	"2:\n\t"						\
 	"mov  %%r9b, %[vector]\n\t"				\
 	"mov  %%r10, %[error_code]\n\t"
@@ -1447,7 +1454,7 @@ static inline bool kvm_is_pmu_enabled(void)
 
 static inline bool kvm_is_mediated_pmu_enabled(void)
 {
-	if (host_cpu_is_intel)
+	if (host_cpu_is_intel_compatible)
 		return get_kvm_intel_param_bool("enable_mediated_pmu");
 
 	return get_kvm_amd_param_bool("enable_mediated_pmu");

tools/testing/selftests/kvm/include/x86/smm.h

@@ -8,7 +8,7 @@
 #define SMRAM_MEMSLOT	((1 << 16) | 1)
 #define SMRAM_PAGES	(SMRAM_SIZE / PAGE_SIZE)
 
-void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa,
+void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa,
 		 const void *smi_handler, size_t handler_size);
 
 void inject_smi(struct kvm_vcpu *vcpu);

tools/testing/selftests/kvm/include/x86/vmx.h

@@ -309,125 +309,67 @@ struct vmx_msr_entry {
 
 #include "evmcs.h"
 
-static inline int vmxon(u64 phys)
+#define GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, __r, __pa)		\
-{
+	__GUEST_ASSERT(!__r, __stringify(insn) "[0x%lx] hit %s",	\
-	u8 ret;
+		       __pa, __r < 0 ? "VM-Fail" : ex_str(__r))
 
-	__asm__ __volatile__ ("vmxon %[pa]; setna %[ret]"
+#define BUILD_VMCS_ASM_HELPERS(insn)					\
-		: [ret]"=rm"(ret)
+static inline int __##insn(u64 vmcs_pa)					\
-		: [pa]"m"(phys)
+{									\
-		: "cc", "memory");
+	u64 error_code;							\
-
+	u8 vector;							\
-	return ret;
+	u8 failed;							\
+									\
+	asm volatile(KVM_ASM_SAFE(__stringify(insn) " %[pa]")		\
+		     "\n\tsetna %[failed]"				\
+		     : KVM_ASM_SAFE_OUTPUTS(vector, error_code),	\
+		       [failed]"=qm"(failed)				\
+		     : [pa]"m"(vmcs_pa)					\
+		     : "cc", "memory", KVM_ASM_SAFE_CLOBBERS);		\
+									\
+	return vector ? vector : failed ? -EINVAL : 0;			\
+}									\
+									\
+static inline void insn(u64 vmcs_pa)					\
+{									\
+	int ret = __##insn(vmcs_pa);					\
+									\
+	GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, ret, vmcs_pa);		\
 }
 
+BUILD_VMCS_ASM_HELPERS(vmxon)
+BUILD_VMCS_ASM_HELPERS(vmptrld)
+BUILD_VMCS_ASM_HELPERS(vmclear)
+
 static inline void vmxoff(void)
 {
 	__asm__ __volatile__("vmxoff");
 }
 
-static inline int vmclear(u64 vmcs_pa)
+static inline u64 vmptrst(void)
-{
-	u8 ret;
-
-	__asm__ __volatile__ ("vmclear %[pa]; setna %[ret]"
-		: [ret]"=rm"(ret)
-		: [pa]"m"(vmcs_pa)
-		: "cc", "memory");
-
-	return ret;
-}
-
-static inline int vmptrld(u64 vmcs_pa)
-{
-	u8 ret;
-
-	if (enable_evmcs)
-		return -1;
-
-	__asm__ __volatile__ ("vmptrld %[pa]; setna %[ret]"
-		: [ret]"=rm"(ret)
-		: [pa]"m"(vmcs_pa)
-		: "cc", "memory");
-
-	return ret;
-}
-
-static inline int vmptrst(u64 *value)
-{
-	u64 tmp;
-	u8 ret;
-
-	if (enable_evmcs)
-		return evmcs_vmptrst(value);
-
-	__asm__ __volatile__("vmptrst %[value]; setna %[ret]"
-		: [value]"=m"(tmp), [ret]"=rm"(ret)
-		: : "cc", "memory");
-
-	*value = tmp;
-	return ret;
-}
-
-/*
- * A wrapper around vmptrst that ignores errors and returns zero if the
- * vmptrst instruction fails.
- */
-static inline u64 vmptrstz(void)
 {
 	u64 value = 0;
-	vmptrst(&value);
+	u8 ret;
+
+	__asm__ __volatile__("vmptrst %[value]; setna %[ret]"
+		: [value]"=m"(value), [ret]"=rm"(ret)
+		: : "cc", "memory");
+
+	__GUEST_ASSERT(!ret, "vmptrst failed");
 	return value;
 }
 
-static inline int vmlaunch(void)
+int __vmlaunch(void);
+int __vmresume(void);
+
+static inline void vmlaunch(void)
 {
-	int ret;
+	__GUEST_ASSERT(!__vmlaunch(), "vmlaunch hit VM-Fail");
-
-	if (enable_evmcs)
-		return evmcs_vmlaunch();
-
-	__asm__ __volatile__("push $0;"
-			     "vmwrite %%rsp, %[host_rsp];"
-			     "lea 1f(%%rip), %%rax;"
-			     "vmwrite %%rax, %[host_rip];"
-			     VMX_SWITCH_GPRS_ASM
-			     "vmlaunch;"
-			     "incq (%%rsp);"
-			     "1: ;"
-			     VMX_SWITCH_GPRS_ASM
-			     "pop %%rax;"
-			     : [ret]"=&a"(ret)
-			     : [host_rsp]"r"((u64)HOST_RSP),
-			       [host_rip]"r"((u64)HOST_RIP),
-			       GUEST_REGS_OFFSETS
-			     : "memory", "cc");
-	return ret;
 }
 
-static inline int vmresume(void)
+static inline void vmresume(void)
 {
-	int ret;
+	__GUEST_ASSERT(!__vmresume(), "vmresume hit VM-Fail");
-
-	if (enable_evmcs)
-		return evmcs_vmresume();
-
-	__asm__ __volatile__("push $0;"
-			     "vmwrite %%rsp, %[host_rsp];"
-			     "lea 1f(%%rip), %%rax;"
-			     "vmwrite %%rax, %[host_rip];"
-			     VMX_SWITCH_GPRS_ASM
-			     "vmresume;"
-			     "incq (%%rsp);"
-			     "1: ;"
-			     VMX_SWITCH_GPRS_ASM
-			     "pop %%rax;"
-			     : [ret]"=&a"(ret)
-			     : [host_rsp]"r"((u64)HOST_RSP),
-			       [host_rip]"r"((u64)HOST_RIP),
-			       GUEST_REGS_OFFSETS
-			     : "memory", "cc");
-	return ret;
 }
 
 static inline void vmcall(void)
@@ -444,7 +386,7 @@ static inline void vmcall(void)
 			       "r10", "r11", "r12", "r13", "r14", "r15");
 }
 
-static inline int vmread(u64 encoding, u64 *value)
+static inline int __vmread(u64 encoding, u64 *value)
 {
 	u64 tmp;
 	u8 ret;
@@ -457,22 +399,21 @@ static inline int vmread(u64 encoding, u64 *value)
 		: [encoding]"r"(encoding)
 		: "cc", "memory");
 
-	*value = tmp;
+	if (!ret)
+		*value = tmp;
 	return ret;
 }
 
-/*
+static inline u64 vmread(u64 encoding)
- * A wrapper around vmread that ignores errors and returns zero if the
- * vmread instruction fails.
- */
-static inline u64 vmreadz(u64 encoding)
 {
 	u64 value = 0;
-	vmread(encoding, &value);
+
+	__GUEST_ASSERT(!__vmread(encoding, &value),
+		       "vmread[0x%lx] hit VM-Fail", encoding);
 	return value;
 }
 
-static inline int vmwrite(u64 encoding, u64 value)
+static __always_inline int __vmwrite(u64 encoding, u64 value)
 {
 	u8 ret;
 
@@ -487,6 +428,12 @@ static inline int vmwrite(u64 encoding, u64 value)
 	return ret;
 }
 
+static inline void vmwrite(u64 encoding, u64 value)
+{
+	__GUEST_ASSERT(!__vmwrite(encoding, value),
+		       "vmwrite[0x%lx] = 0x%lx hit VM-Fail", encoding, value);
+}
+
 static inline u32 vmcs_revision(void)
 {
 	return rdmsr(MSR_IA32_VMX_BASIC);
@@ -550,13 +497,24 @@ union vmx_ctrl_msr {
 };
 
 struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva);
-bool prepare_for_vmx_operation(struct vmx_pages *vmx);
+void prepare_for_vmx_operation(struct vmx_pages *vmx);
 void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip);
-bool load_vmcs(struct vmx_pages *vmx);
+void load_vmcs(struct vmx_pages *vmx);
 
 bool ept_1g_pages_supported(void);
 
-bool kvm_cpu_has_ept(void);
+bool kvm_cpu_has_secondary_exec_control(u32 ctrl);
+
+static inline bool kvm_cpu_has_ept(void)
+{
+	return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_ENABLE_EPT);
+}
+
+static inline bool kvm_cpu_has_vmx_virtualize_apic_accesses(void)
+{
+	return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
+}
+
 void vm_enable_ept(struct kvm_vm *vm);
 void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm);
 

tools/testing/selftests/kvm/lib/arm64/processor.c

@@ -115,9 +115,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
 	if (vm->mmu.pgd_created)
 		return;
 
-	vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages,
+	vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages);
-					 KVM_GUEST_PAGE_TABLE_MIN_PADDR,
-					 vm->memslots[MEM_REGION_PT]);
 	vm->mmu.pgd_created = true;
 }
 

tools/testing/selftests/kvm/lib/elf.c

@@ -12,6 +12,44 @@
 
 #include "kvm_util.h"
 
+static ssize_t elf_read(int fd, void *buf, size_t count)
+{
+	ssize_t rc;
+	ssize_t num_read = 0;
+	size_t num_left = count;
+	char *ptr = buf;
+
+	TEST_ASSERT(count, "Count must be non-zero");
+
+	do {
+		rc = read(fd, ptr, num_left);
+
+		switch (rc) {
+		case -1:
+			TEST_ASSERT(errno == EAGAIN || errno == EINTR,
+				    "Unexpected read failure,\n"
+				    "  rc: %zi errno: %i", rc, errno);
+			break;
+
+		case 0:
+			TEST_FAIL("Unexpected EOF,\n"
+				  "   rc: %zi num_read: %zi num_left: %zu",
+				  rc, num_read, num_left);
+			break;
+
+		default:
+			TEST_ASSERT(rc > 0, "Unexpected ret from read,\n"
+				    "  rc: %zi errno: %i", rc, errno);
+			num_read += rc;
+			num_left -= rc;
+			ptr += rc;
+			break;
+		}
+	} while (num_read < count);
+
+	return num_read;
+}
+
 static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp)
 {
 	off_t offset_rv;
@@ -31,7 +69,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp)
 	 * the real size of the ELF header.
 	 */
 	unsigned char ident[EI_NIDENT];
-	test_read(fd, ident, sizeof(ident));
+	elf_read(fd, ident, sizeof(ident));
 	TEST_ASSERT((ident[EI_MAG0] == ELFMAG0) && (ident[EI_MAG1] == ELFMAG1)
 		&& (ident[EI_MAG2] == ELFMAG2) && (ident[EI_MAG3] == ELFMAG3),
 		"ELF MAGIC Mismatch,\n"
@@ -79,7 +117,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp)
 	offset_rv = lseek(fd, 0, SEEK_SET);
 	TEST_ASSERT(offset_rv == 0, "Seek to ELF header failed,\n"
 		"  rv: %zi expected: %i", offset_rv, 0);
-	test_read(fd, hdrp, sizeof(*hdrp));
+	elf_read(fd, hdrp, sizeof(*hdrp));
 	TEST_ASSERT(hdrp->e_phentsize == sizeof(Elf64_Phdr),
 		"Unexpected physical header size,\n"
 		"  hdrp->e_phentsize: %x\n"
@@ -146,7 +184,7 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename)
 
 		/* Read in the program header. */
 		Elf64_Phdr phdr;
-		test_read(fd, &phdr, sizeof(phdr));
+		elf_read(fd, &phdr, sizeof(phdr));
 
 		/* Skip if this header doesn't describe a loadable segment. */
 		if (phdr.p_type != PT_LOAD)
@@ -186,8 +224,8 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename)
 				"  expected: 0x%jx",
 				n1, errno, (intmax_t) offset_rv,
 				(intmax_t) phdr.p_offset);
-			test_read(fd, addr_gva2hva(vm, phdr.p_vaddr),
+			elf_read(fd, addr_gva2hva(vm, phdr.p_vaddr),
-				phdr.p_filesz);
+				 phdr.p_filesz);
 		}
 	}
 	close(fd);

tools/testing/selftests/kvm/lib/guest_sprintf.c

@@ -300,15 +300,3 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args)
 	*str = '\0';
 	return str - buf;
 }
-
-int guest_snprintf(char *buf, int n, const char *fmt, ...)
-{
-	va_list va;
-	int len;
-
-	va_start(va, fmt);
-	len = guest_vsnprintf(buf, n, fmt, va);
-	va_end(va);
-
-	return len;
-}

tools/testing/selftests/kvm/lib/io.c

@@ -1,157 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * tools/testing/selftests/kvm/lib/io.c
- *
- * Copyright (C) 2018, Google LLC.
- */
-
-#include "test_util.h"
-
-/* Test Write
- *
- * A wrapper for write(2), that automatically handles the following
- * special conditions:
- *
- *   + Interrupted system call (EINTR)
- *   + Write of less than requested amount
- *   + Non-block return (EAGAIN)
- *
- * For each of the above, an additional write is performed to automatically
- * continue writing the requested data.
- * There are also many cases where write(2) can return an unexpected
- * error (e.g. EIO).  Such errors cause a TEST_ASSERT failure.
- *
- * Note, for function signature compatibility with write(2), this function
- * returns the number of bytes written, but that value will always be equal
- * to the number of requested bytes.  All other conditions in this and
- * future enhancements to this function either automatically issue another
- * write(2) or cause a TEST_ASSERT failure.
- *
- * Args:
- *  fd    - Opened file descriptor to file to be written.
- *  count - Number of bytes to write.
- *
- * Output:
- *  buf   - Starting address of data to be written.
- *
- * Return:
- *  On success, number of bytes written.
- *  On failure, a TEST_ASSERT failure is caused.
- */
-ssize_t test_write(int fd, const void *buf, size_t count)
-{
-	ssize_t rc;
-	ssize_t num_written = 0;
-	size_t num_left = count;
-	const char *ptr = buf;
-
-	/* Note: Count of zero is allowed (see "RETURN VALUE" portion of
-	 * write(2) manpage for details.
-	 */
-	TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count);
-
-	do {
-		rc = write(fd, ptr, num_left);
-
-		switch (rc) {
-		case -1:
-			TEST_ASSERT(errno == EAGAIN || errno == EINTR,
-				    "Unexpected write failure,\n"
-				    "  rc: %zi errno: %i", rc, errno);
-			continue;
-
-		case 0:
-			TEST_FAIL("Unexpected EOF,\n"
-				  "  rc: %zi num_written: %zi num_left: %zu",
-				  rc, num_written, num_left);
-			break;
-
-		default:
-			TEST_ASSERT(rc >= 0, "Unexpected ret from write,\n"
-				"  rc: %zi errno: %i", rc, errno);
-			num_written += rc;
-			num_left -= rc;
-			ptr += rc;
-			break;
-		}
-	} while (num_written < count);
-
-	return num_written;
-}
-
-/* Test Read
- *
- * A wrapper for read(2), that automatically handles the following
- * special conditions:
- *
- *   + Interrupted system call (EINTR)
- *   + Read of less than requested amount
- *   + Non-block return (EAGAIN)
- *
- * For each of the above, an additional read is performed to automatically
- * continue reading the requested data.
- * There are also many cases where read(2) can return an unexpected
- * error (e.g. EIO).  Such errors cause a TEST_ASSERT failure.  Note,
- * it is expected that the file opened by fd at the current file position
- * contains at least the number of requested bytes to be read.  A TEST_ASSERT
- * failure is produced if an End-Of-File condition occurs, before all the
- * data is read.  It is the callers responsibility to assure that sufficient
- * data exists.
- *
- * Note, for function signature compatibility with read(2), this function
- * returns the number of bytes read, but that value will always be equal
- * to the number of requested bytes.  All other conditions in this and
- * future enhancements to this function either automatically issue another
- * read(2) or cause a TEST_ASSERT failure.
- *
- * Args:
- *  fd    - Opened file descriptor to file to be read.
- *  count - Number of bytes to read.
- *
- * Output:
- *  buf   - Starting address of where to write the bytes read.
- *
- * Return:
- *  On success, number of bytes read.
- *  On failure, a TEST_ASSERT failure is caused.
- */
-ssize_t test_read(int fd, void *buf, size_t count)
-{
-	ssize_t rc;
-	ssize_t num_read = 0;
-	size_t num_left = count;
-	char *ptr = buf;
-
-	/* Note: Count of zero is allowed (see "If count is zero" portion of
-	 * read(2) manpage for details.
-	 */
-	TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count);
-
-	do {
-		rc = read(fd, ptr, num_left);
-
-		switch (rc) {
-		case -1:
-			TEST_ASSERT(errno == EAGAIN || errno == EINTR,
-				    "Unexpected read failure,\n"
-				    "  rc: %zi errno: %i", rc, errno);
-			break;
-
-		case 0:
-			TEST_FAIL("Unexpected EOF,\n"
-				  "   rc: %zi num_read: %zi num_left: %zu",
-				  rc, num_read, num_left);
-			break;
-
-		default:
-			TEST_ASSERT(rc > 0, "Unexpected ret from read,\n"
-				    "  rc: %zi errno: %i", rc, errno);
-			num_read += rc;
-			num_left -= rc;
-			ptr += rc;
-			break;
-		}
-	} while (num_read < count);
-
-	return num_read;
-}

tools/testing/selftests/kvm/lib/kvm_util.c

@@ -17,6 +17,7 @@
 #include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>
+#include <linux/align.h>
 #include <linux/kernel.h>
 
 #include "../../../../../mm/gup_test.h"
@@ -291,6 +292,7 @@ __weak void vm_populate_gva_bitmap(struct kvm_vm *vm)
 struct kvm_vm *____vm_create(struct vm_shape shape)
 {
 	struct kvm_vm *vm;
+	int i;
 
 	vm = calloc(1, sizeof(*vm));
 	TEST_ASSERT(vm != NULL, "Insufficient Memory");
@@ -299,6 +301,8 @@ struct kvm_vm *____vm_create(struct vm_shape shape)
 	vm->regions.gpa_tree = RB_ROOT;
 	vm->regions.hva_tree = RB_ROOT;
 	hash_init(vm->regions.slot_hash);
+	for (i = 0; i < NR_MEM_REGIONS; i++)
+		vm->memslots[i] = KVM_INVALID_MEMSLOT;
 
 	vm->mode = shape.mode;
 	vm->type = shape.type;
@@ -496,7 +500,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
 	struct userspace_mem_region *slot0;
 	u64 gmem_flags = 0;
 	struct kvm_vm *vm;
-	int i, flags;
+	int flags;
 
 	kvm_set_files_rlimit(nr_runnable_vcpus);
 
@@ -524,8 +528,10 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
 	}
 
 	vm_mem_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags, -1, 0, gmem_flags);
-	for (i = 0; i < NR_MEM_REGIONS; i++)
+	____vm_override_mem_region(vm, MEM_REGION_CODE, 0);
-		vm->memslots[i] = 0;
+	____vm_override_mem_region(vm, MEM_REGION_PT, 0);
+	____vm_override_mem_region(vm, MEM_REGION_DATA, 0);
+	____vm_override_mem_region(vm, MEM_REGION_TEST_DATA, 0);
 
 	kvm_vm_elf_load(vm, program_invocation_name);
 
@@ -1249,6 +1255,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot)
 {
 	struct userspace_mem_region *region;
 
+	TEST_ASSERT(memslot != KVM_INVALID_MEMSLOT, "vm->memslots[] unpopulated?");
+
 	hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
 			       memslot)
 		if (region->region.slot == memslot)
@@ -1518,9 +1526,7 @@ static gva_t ____vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva,
 	u64 pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
 
 	virt_pgd_alloc(vm);
-	gpa_t gpa = __vm_phy_pages_alloc(vm, pages,
+	gpa_t gpa = __vm_phy_pages_alloc(vm, pages, type, protected);
-					   KVM_UTIL_MIN_PFN * vm->page_size,
-					   vm->memslots[type], protected);
 
 	/*
 	 * Find an unused range of virtual page addresses of at least
@@ -1796,6 +1802,7 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu)
 	TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
 
 	reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
+	TEST_ASSERT(reg_list, "Failed to allocate reg_list");
 	reg_list->n = reg_list_n.n;
 	vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list);
 	return reg_list;
@@ -2094,33 +2101,21 @@ const char *exit_reason_str(unsigned int exit_reason)
 }
 
 /*
- * Physical Contiguous Page Allocator
+ * Allocate contiguous (guest) physical pages in a given memory region, at or
+ * above the minimum specified GPA.  If the memory is protected/private, also
+ * add the allocated pages to the region's set of protected pages, e.g. so that
+ * arch code knows which pages need to be encrypted when launching the VM.
  *
- * Input Args:
+ * Note, success is guaranteed!
- *   vm - Virtual Machine
- *   num - number of pages
- *   min_gpa - Physical address minimum
- *   memslot - Memory region to allocate page from
- *   protected - True if the pages will be used as protected/private memory
- *
- * Output Args: None
- *
- * Return:
- *   Starting physical address
- *
- * Within the VM specified by vm, locates a range of available physical
- * pages at or above min_gpa. If found, the pages are marked as in use
- * and their base address is returned. A TEST_ASSERT failure occurs if
- * not enough pages are available at or above min_gpa.
  */
-gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
+gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa,
-			   gpa_t min_gpa, u32 memslot,
+			     u32 memslot, bool protected, bool naturally_aligned)
-			   bool protected)
 {
+	size_t alignment = naturally_aligned ? nr_pages : 1;
 	struct userspace_mem_region *region;
 	sparsebit_idx_t pg, base;
 
-	TEST_ASSERT(num > 0, "Must allocate at least one page");
+	TEST_ASSERT(nr_pages, "Must allocate at least one page");
 
 	TEST_ASSERT((min_gpa % vm->page_size) == 0, "Min physical address "
 		"not divisible by page size.\n"
@@ -2131,43 +2126,78 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
 	TEST_ASSERT(!protected || region->protected_phy_pages,
 		    "Region doesn't support protected memory");
 
-	base = pg = min_gpa >> vm->page_shift;
+	base = min_gpa >> vm->page_shift;
-	do {
+again:
-		for (; pg < base + num; ++pg) {
+	base = ALIGN(base, alignment);
-			if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
+	for (pg = base; pg < base + nr_pages; ++pg) {
-				base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
+		if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
-				break;
+			base = sparsebit_next_set(region->unused_phy_pages, pg);
-			}
+			if (!base)
+				goto enomem;
+			goto again;
 		}
-	} while (pg && pg != base + num);
-
-	if (pg == 0) {
-		fprintf(stderr, "No guest physical page available, "
-			"min_gpa: 0x%lx page_size: 0x%x memslot: %u\n",
-			min_gpa, vm->page_size, memslot);
-		fputs("---- vm dump ----\n", stderr);
-		vm_dump(stderr, vm, 2);
-		abort();
 	}
 
-	for (pg = base; pg < base + num; ++pg) {
+	for (pg = base; pg < base + nr_pages; ++pg) {
 		sparsebit_clear(region->unused_phy_pages, pg);
 		if (protected)
 			sparsebit_set(region->protected_phy_pages, pg);
 	}
 
 	return base * vm->page_size;
+
+enomem:
+	fprintf(stderr, "No guest physical page available, min_gpa: 0x%lx page_size: 0x%x memslot: %u\n",
+		min_gpa, vm->page_size, memslot);
+	fputs("---- vm dump ----\n", stderr);
+	vm_dump(stderr, vm, 2);
+	abort();
+	__builtin_unreachable();
 }
 
-gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot)
+__weak bool kvm_arch_needs_naturally_aligned_page_tables(void)
 {
-	return vm_phy_pages_alloc(vm, 1, min_gpa, memslot);
+	return false;
 }
 
-gpa_t vm_alloc_page_table(struct kvm_vm *vm)
+gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages,
+			   enum kvm_mem_region_type type, bool protected)
 {
-	return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
+	struct userspace_mem_region *region = vm_get_mem_region(vm, type);
-				 vm->memslots[MEM_REGION_PT]);
+	bool naturally_aligned = false;
+	gpa_t min_gpa;
+
+	TEST_ASSERT(region, "No region for type '%u', memslot '%u'",
+		    type, vm->memslots[type]);
+
+	switch (type) {
+	case MEM_REGION_CODE:
+	case MEM_REGION_DATA:
+	case MEM_REGION_TEST_DATA:
+		/*
+		 * If the region is backed by the default memslot (id=0), use
+		 * selftests' hardcoded minimum PFN, otherwise use the base of
+		 * the custom memory slot that backs the region.
+		 */
+		if (!vm->memslots[type])
+			min_gpa = KVM_UTIL_MIN_PFN * vm->page_size;
+		else
+			min_gpa = region->region.guest_phys_addr;
+		break;
+	case MEM_REGION_PT:
+		min_gpa = KVM_GUEST_PAGE_TABLE_MIN_PADDR;
+		naturally_aligned = kvm_arch_needs_naturally_aligned_page_tables();
+		break;
+	case MEM_REGION_TEST_EXTRA:
+		min_gpa = region->region.guest_phys_addr;
+		break;
+	default:
+		TEST_FAIL("Invalid memory region type '%u'", type);
+		break;
+	}
+
+	return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, vm->memslots[type],
+				      protected, naturally_aligned);
 }
 
 /*

tools/testing/selftests/kvm/lib/loongarch/processor.c

@@ -9,7 +9,6 @@
 #include "processor.h"
 #include "ucall_common.h"
 
-#define LOONGARCH_PAGE_TABLE_PHYS_MIN		0x200000
 #define LOONGARCH_GUEST_STACK_VADDR_MIN		0x200000
 
 static gpa_t invalid_pgtable[4];
@@ -57,9 +56,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
 	child = table = 0;
 	for (i = 0; i < vm->mmu.pgtable_levels; i++) {
 		invalid_pgtable[i] = child;
-		table = vm_phy_page_alloc(vm, LOONGARCH_PAGE_TABLE_PHYS_MIN,
+		table = vm_alloc_page_table(vm);
-				vm->memslots[MEM_REGION_PT]);
-		TEST_ASSERT(table, "Fail to allocate page tale at level %d\n", i);
 		virt_set_pgtable(vm, table, child);
 		child = table;
 	}

tools/testing/selftests/kvm/lib/riscv/processor.c

@@ -69,9 +69,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
 	if (vm->mmu.pgd_created)
 		return;
 
-	vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages,
+	vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages);
-					 KVM_GUEST_PAGE_TABLE_MIN_PADDR,
-					 vm->memslots[MEM_REGION_PT]);
 	vm->mmu.pgd_created = true;
 }
 

tools/testing/selftests/kvm/lib/s390/processor.c

@@ -20,9 +20,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
 	if (vm->mmu.pgd_created)
 		return;
 
-	gpa = vm_phy_pages_alloc(vm, PAGES_PER_REGION,
+	gpa = vm_alloc_page_table_pages(vm, PAGES_PER_REGION);
-				   KVM_GUEST_PAGE_TABLE_MIN_PADDR,
-				   vm->memslots[MEM_REGION_PT]);
 	memset(addr_gpa2hva(vm, gpa), 0xff, PAGES_PER_REGION * vm->page_size);
 
 	vm->mmu.pgd = gpa;
@@ -38,8 +36,7 @@ static u64 virt_alloc_region(struct kvm_vm *vm, int ri)
 {
 	u64 taddr;
 
-	taddr = vm_phy_pages_alloc(vm,  ri < 4 ? PAGES_PER_REGION : 1,
+	taddr = vm_alloc_page_table_pages(vm, ri < 4 ? PAGES_PER_REGION : 1);
-				   KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
 	memset(addr_gpa2hva(vm, taddr), 0xff, PAGES_PER_REGION * vm->page_size);
 
 	return (taddr & REGION_ENTRY_ORIGIN)

tools/testing/selftests/kvm/lib/userfaultfd_util.c

@@ -119,13 +119,13 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
 	uffd_desc = malloc(sizeof(struct uffd_desc));
 	TEST_ASSERT(uffd_desc, "Failed to malloc uffd descriptor");
 
-	uffd_desc->pipefds = calloc(sizeof(int), num_readers);
+	uffd_desc->pipefds = calloc(num_readers, sizeof(int));
 	TEST_ASSERT(uffd_desc->pipefds, "Failed to alloc pipes");
 
-	uffd_desc->readers = calloc(sizeof(pthread_t), num_readers);
+	uffd_desc->readers = calloc(num_readers, sizeof(pthread_t));
 	TEST_ASSERT(uffd_desc->readers, "Failed to alloc reader threads");
 
-	uffd_desc->reader_args = calloc(sizeof(struct uffd_reader_args), num_readers);
+	uffd_desc->reader_args = calloc(num_readers, sizeof(struct uffd_reader_args));
 	TEST_ASSERT(uffd_desc->reader_args, "Failed to alloc reader_args");
 
 	uffd_desc->num_readers = num_readers;

tools/testing/selftests/kvm/lib/x86/memstress.c

@@ -33,15 +33,15 @@ __asm__(
 static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id)
 {
 	GUEST_ASSERT(vmx->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT(load_vmcs(vmx));
+	load_vmcs(vmx);
 	GUEST_ASSERT(ept_1g_pages_supported());
 
 	*(u64 *)vmx->stack = vcpu_id;
 	prepare_vmcs(vmx, memstress_l2_guest_entry);
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL);
 	GUEST_DONE();
 }
 

tools/testing/selftests/kvm/lib/x86/processor.c

@@ -26,6 +26,8 @@ bool host_cpu_is_amd;
 bool host_cpu_is_intel;
 bool host_cpu_is_hygon;
 bool host_cpu_is_amd_compatible;
+bool host_cpu_is_zhaoxin;
+bool host_cpu_is_intel_compatible;
 bool is_forced_emulation_enabled;
 u64 guest_tsc_khz;
 struct kvm_mmu guest_mmu;
@@ -158,7 +160,7 @@ static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent)
 
 bool kvm_is_tdp_enabled(void)
 {
-	if (host_cpu_is_intel)
+	if (host_cpu_is_intel_compatible)
 		return get_kvm_intel_param_bool("ept");
 	else
 		return get_kvm_amd_param_bool("npt");
@@ -529,7 +531,10 @@ void tdp_identity_map_default_memslots(struct kvm_vm *vm)
 
 	/* Only memslot 0 is mapped here, ensure it's the only one being used */
 	for (s = 0; s < NR_MEM_REGIONS; s++)
-		TEST_ASSERT_EQ(vm->memslots[s], 0);
+		TEST_ASSERT(!vm->memslots[s] ||
+			    vm->memslots[s] == KVM_INVALID_MEMSLOT,
+			    "Unhandled memslot '%u' for type '%u'",
+			    vm->memslots[s], s);
 
 	i = (region->region.guest_phys_addr >> vm->page_shift) - 1;
 	last = i + (region->region.memory_size >> vm->page_shift);
@@ -819,6 +824,8 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus)
 	sync_global_to_guest(vm, host_cpu_is_amd);
 	sync_global_to_guest(vm, host_cpu_is_hygon);
 	sync_global_to_guest(vm, host_cpu_is_amd_compatible);
+	sync_global_to_guest(vm, host_cpu_is_zhaoxin);
+	sync_global_to_guest(vm, host_cpu_is_intel_compatible);
 	sync_global_to_guest(vm, is_forced_emulation_enabled);
 	sync_global_to_guest(vm, pmu_errata_mask);
 
@@ -1444,6 +1451,8 @@ void kvm_selftest_arch_init(void)
 	host_cpu_is_amd = this_cpu_is_amd();
 	host_cpu_is_hygon = this_cpu_is_hygon();
 	host_cpu_is_amd_compatible = host_cpu_is_amd || host_cpu_is_hygon;
+	host_cpu_is_zhaoxin = this_cpu_is_zhaoxin();
+	host_cpu_is_intel_compatible = host_cpu_is_intel || host_cpu_is_zhaoxin;
 	is_forced_emulation_enabled = kvm_is_forced_emulation_enabled();
 
 	kvm_init_pmu_errata();
@@ -1465,13 +1474,13 @@ bool kvm_arch_has_default_irqchip(void)
 	return true;
 }
 
-void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa,
+void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa,
 		 const void *smi_handler, size_t handler_size)
 {
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, smram_gpa,
+	vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS,
-				    SMRAM_MEMSLOT, SMRAM_PAGES, 0);
+			       smram_gpa, SMRAM_MEMSLOT, SMRAM_PAGES);
-	TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa,
+
-				       SMRAM_MEMSLOT) == smram_gpa,
+	TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, MEM_REGION_TEST_EXTRA) == smram_gpa,
 		    "Could not allocate guest physical addresses for SMRAM");
 
 	memset(addr_gpa2hva(vm, smram_gpa), 0x0, SMRAM_SIZE);

tools/testing/selftests/kvm/lib/x86/vmx.c

@@ -125,7 +125,7 @@ vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva)
 	return vmx;
 }
 
-bool prepare_for_vmx_operation(struct vmx_pages *vmx)
+void prepare_for_vmx_operation(struct vmx_pages *vmx)
 {
 	u64 feature_control;
 	u64 required;
@@ -163,28 +163,75 @@ bool prepare_for_vmx_operation(struct vmx_pages *vmx)
 
 	/* Enter VMX root operation. */
 	*(u32 *)(vmx->vmxon) = vmcs_revision();
-	if (vmxon(vmx->vmxon_gpa))
+	vmxon(vmx->vmxon_gpa);
-		return false;
-
-	return true;
 }
 
-bool load_vmcs(struct vmx_pages *vmx)
+void load_vmcs(struct vmx_pages *vmx)
 {
 	/* Load a VMCS. */
 	*(u32 *)(vmx->vmcs) = vmcs_revision();
-	if (vmclear(vmx->vmcs_gpa))
+	vmclear(vmx->vmcs_gpa);
-		return false;
 
-	if (vmptrld(vmx->vmcs_gpa))
+	vmptrld(vmx->vmcs_gpa);
-		return false;
 
 	/* Setup shadow VMCS, do not load it yet. */
 	*(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul;
-	if (vmclear(vmx->shadow_vmcs_gpa))
+	vmclear(vmx->shadow_vmcs_gpa);
-		return false;
+}
 
-	return true;
+#define __BUILD_VMX_VM_ENTRY_HELPER(insn, prefix, vmwrite_insn, vmwrite_operand,	\
+				    __host_rsp, __host_rip)				\
+static int __##prefix##_##insn(void)							\
+{											\
+	int ret;									\
+											\
+	__asm__ __volatile__("push $0;"							\
+			     __stringify(vmwrite_insn) " %%rsp, %[host_rsp];"		\
+			     "lea 1f(%%rip), %%rax;"					\
+			     __stringify(vmwrite_insn) " %%rax, %[host_rip];"		\
+			     VMX_SWITCH_GPRS_ASM					\
+			     __stringify(insn)";"					\
+			     "incq (%%rsp);"						\
+			     "1: ;"							\
+			     VMX_SWITCH_GPRS_ASM					\
+			     "pop %%rax;"						\
+			     : [ret]"=&a"(ret)						\
+			     : [host_rsp]__stringify(vmwrite_operand)(__host_rsp),	\
+			       [host_rip]__stringify(vmwrite_operand)(__host_rip),	\
+			       GUEST_REGS_OFFSETS					\
+			     : "memory", "cc");						\
+	return ret;									\
+}
+
+#define BUILD_VMX_VM_ENTRY_HELPER(insn) \
+	__BUILD_VMX_VM_ENTRY_HELPER(insn, _, vmwrite, r, (u64)HOST_RSP, (u64)HOST_RIP)	\
+	__BUILD_VMX_VM_ENTRY_HELPER(insn, __evmcs, mov, m,				\
+				    current_evmcs->host_rsp, current_evmcs->host_rip)
+
+BUILD_VMX_VM_ENTRY_HELPER(vmlaunch)
+BUILD_VMX_VM_ENTRY_HELPER(vmresume)
+
+int __vmlaunch(void)
+{
+	if (enable_evmcs) {
+		current_evmcs->hv_clean_fields = 0;
+		return ____evmcs_vmlaunch();
+	}
+
+	return ____vmlaunch();
+}
+
+int __vmresume(void)
+{
+	if (enable_evmcs) {
+		/* HOST_RIP */
+		current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1;
+		/* HOST_RSP */
+		current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER;
+		return ____evmcs_vmresume();
+	}
+
+	return ____vmresume();
 }
 
 static bool ept_vpid_cap_supported(u64 mask)
@@ -204,8 +251,8 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
 {
 	u32 sec_exec_ctl = 0;
 
-	vmwrite(VIRTUAL_PROCESSOR_ID, 0);
+	__vmwrite(VIRTUAL_PROCESSOR_ID, 0);
-	vmwrite(POSTED_INTR_NV, 0);
+	__vmwrite(POSTED_INTR_NV, 0);
 
 	vmwrite(PIN_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PINBASED_CTLS));
 
@@ -222,7 +269,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
 		sec_exec_ctl |= SECONDARY_EXEC_ENABLE_EPT;
 	}
 
-	if (!vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl))
+	if (!__vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl))
 		vmwrite(CPU_BASED_VM_EXEC_CONTROL,
 			rdmsr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS);
 	else {
@@ -242,16 +289,16 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
 		VM_ENTRY_IA32E_MODE);		  /* 64-bit guest */
 	vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0);
 	vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0);
-	vmwrite(TPR_THRESHOLD, 0);
+	__vmwrite(TPR_THRESHOLD, 0);
 
 	vmwrite(CR0_GUEST_HOST_MASK, 0);
 	vmwrite(CR4_GUEST_HOST_MASK, 0);
 	vmwrite(CR0_READ_SHADOW, get_cr0());
 	vmwrite(CR4_READ_SHADOW, get_cr4());
 
-	vmwrite(MSR_BITMAP, vmx->msr_gpa);
+	__vmwrite(MSR_BITMAP, vmx->msr_gpa);
-	vmwrite(VMREAD_BITMAP, vmx->vmread_gpa);
+	__vmwrite(VMREAD_BITMAP, vmx->vmread_gpa);
-	vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa);
+	__vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa);
 }
 
 /*
@@ -261,7 +308,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
  */
 static inline void init_vmcs_host_state(void)
 {
-	u32 exit_controls = vmreadz(VM_EXIT_CONTROLS);
+	u32 exit_controls = vmread(VM_EXIT_CONTROLS);
 
 	vmwrite(HOST_ES_SELECTOR, get_es());
 	vmwrite(HOST_CS_SELECTOR, get_cs());
@@ -301,23 +348,23 @@ static inline void init_vmcs_host_state(void)
  */
 static inline void init_vmcs_guest_state(void *rip, void *rsp)
 {
-	vmwrite(GUEST_ES_SELECTOR, vmreadz(HOST_ES_SELECTOR));
+	vmwrite(GUEST_ES_SELECTOR, vmread(HOST_ES_SELECTOR));
-	vmwrite(GUEST_CS_SELECTOR, vmreadz(HOST_CS_SELECTOR));
+	vmwrite(GUEST_CS_SELECTOR, vmread(HOST_CS_SELECTOR));
-	vmwrite(GUEST_SS_SELECTOR, vmreadz(HOST_SS_SELECTOR));
+	vmwrite(GUEST_SS_SELECTOR, vmread(HOST_SS_SELECTOR));
-	vmwrite(GUEST_DS_SELECTOR, vmreadz(HOST_DS_SELECTOR));
+	vmwrite(GUEST_DS_SELECTOR, vmread(HOST_DS_SELECTOR));
-	vmwrite(GUEST_FS_SELECTOR, vmreadz(HOST_FS_SELECTOR));
+	vmwrite(GUEST_FS_SELECTOR, vmread(HOST_FS_SELECTOR));
-	vmwrite(GUEST_GS_SELECTOR, vmreadz(HOST_GS_SELECTOR));
+	vmwrite(GUEST_GS_SELECTOR, vmread(HOST_GS_SELECTOR));
 	vmwrite(GUEST_LDTR_SELECTOR, 0);
-	vmwrite(GUEST_TR_SELECTOR, vmreadz(HOST_TR_SELECTOR));
+	vmwrite(GUEST_TR_SELECTOR, vmread(HOST_TR_SELECTOR));
-	vmwrite(GUEST_INTR_STATUS, 0);
+	__vmwrite(GUEST_INTR_STATUS, 0);
-	vmwrite(GUEST_PML_INDEX, 0);
+	__vmwrite(GUEST_PML_INDEX, 0);
 
 	vmwrite(VMCS_LINK_POINTER, -1ll);
 	vmwrite(GUEST_IA32_DEBUGCTL, 0);
-	vmwrite(GUEST_IA32_PAT, vmreadz(HOST_IA32_PAT));
+	__vmwrite(GUEST_IA32_PAT, vmread(HOST_IA32_PAT));
-	vmwrite(GUEST_IA32_EFER, vmreadz(HOST_IA32_EFER));
+	__vmwrite(GUEST_IA32_EFER, vmread(HOST_IA32_EFER));
-	vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL,
+	__vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL,
-		vmreadz(HOST_IA32_PERF_GLOBAL_CTRL));
+		  vmread(HOST_IA32_PERF_GLOBAL_CTRL));
 
 	vmwrite(GUEST_ES_LIMIT, -1);
 	vmwrite(GUEST_CS_LIMIT, -1);
@@ -330,42 +377,42 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp)
 	vmwrite(GUEST_GDTR_LIMIT, 0xffff);
 	vmwrite(GUEST_IDTR_LIMIT, 0xffff);
 	vmwrite(GUEST_ES_AR_BYTES,
-		vmreadz(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093);
+		vmread(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093);
 	vmwrite(GUEST_CS_AR_BYTES, 0xa09b);
 	vmwrite(GUEST_SS_AR_BYTES, 0xc093);
 	vmwrite(GUEST_DS_AR_BYTES,
-		vmreadz(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093);
+		vmread(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093);
 	vmwrite(GUEST_FS_AR_BYTES,
-		vmreadz(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093);
+		vmread(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093);
 	vmwrite(GUEST_GS_AR_BYTES,
-		vmreadz(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093);
+		vmread(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093);
 	vmwrite(GUEST_LDTR_AR_BYTES, 0x10000);
 	vmwrite(GUEST_TR_AR_BYTES, 0x8b);
 	vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0);
 	vmwrite(GUEST_ACTIVITY_STATE, 0);
-	vmwrite(GUEST_SYSENTER_CS, vmreadz(HOST_IA32_SYSENTER_CS));
+	vmwrite(GUEST_SYSENTER_CS, vmread(HOST_IA32_SYSENTER_CS));
-	vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0);
+	__vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0);
 
-	vmwrite(GUEST_CR0, vmreadz(HOST_CR0));
+	vmwrite(GUEST_CR0, vmread(HOST_CR0));
-	vmwrite(GUEST_CR3, vmreadz(HOST_CR3));
+	vmwrite(GUEST_CR3, vmread(HOST_CR3));
-	vmwrite(GUEST_CR4, vmreadz(HOST_CR4));
+	vmwrite(GUEST_CR4, vmread(HOST_CR4));
 	vmwrite(GUEST_ES_BASE, 0);
 	vmwrite(GUEST_CS_BASE, 0);
 	vmwrite(GUEST_SS_BASE, 0);
 	vmwrite(GUEST_DS_BASE, 0);
-	vmwrite(GUEST_FS_BASE, vmreadz(HOST_FS_BASE));
+	vmwrite(GUEST_FS_BASE, vmread(HOST_FS_BASE));
-	vmwrite(GUEST_GS_BASE, vmreadz(HOST_GS_BASE));
+	vmwrite(GUEST_GS_BASE, vmread(HOST_GS_BASE));
 	vmwrite(GUEST_LDTR_BASE, 0);
-	vmwrite(GUEST_TR_BASE, vmreadz(HOST_TR_BASE));
+	vmwrite(GUEST_TR_BASE, vmread(HOST_TR_BASE));
-	vmwrite(GUEST_GDTR_BASE, vmreadz(HOST_GDTR_BASE));
+	vmwrite(GUEST_GDTR_BASE, vmread(HOST_GDTR_BASE));
-	vmwrite(GUEST_IDTR_BASE, vmreadz(HOST_IDTR_BASE));
+	vmwrite(GUEST_IDTR_BASE, vmread(HOST_IDTR_BASE));
 	vmwrite(GUEST_DR7, 0x400);
 	vmwrite(GUEST_RSP, (u64)rsp);
 	vmwrite(GUEST_RIP, (u64)rip);
 	vmwrite(GUEST_RFLAGS, X86_EFLAGS_FIXED);
 	vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0);
-	vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP));
+	vmwrite(GUEST_SYSENTER_ESP, vmread(HOST_IA32_SYSENTER_ESP));
-	vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP));
+	vmwrite(GUEST_SYSENTER_EIP, vmread(HOST_IA32_SYSENTER_EIP));
 }
 
 void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip)
@@ -375,19 +422,19 @@ void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip)
 	init_vmcs_guest_state(guest_rip, vmx->stack);
 }
 
-bool kvm_cpu_has_ept(void)
+bool kvm_cpu_has_secondary_exec_control(u32 ctrl)
 {
-	u64 ctrl;
+	u64 ctrl_msr;
 
 	if (!kvm_cpu_has(X86_FEATURE_VMX))
 		return false;
 
-	ctrl = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32;
+	ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32;
-	if (!(ctrl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
+	if (!(ctrl_msr & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
 		return false;
 
-	ctrl = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32;
+	ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32;
-	return ctrl & SECONDARY_EXEC_ENABLE_EPT;
+	return ctrl_msr & ctrl;
 }
 
 void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm)

tools/testing/selftests/kvm/memslot_perf_test.c

@@ -152,7 +152,7 @@ static void *vcpu_worker(void *__data)
 		case UCALL_SYNC:
 			TEST_ASSERT(uc.args[1] == 0,
 				"Unexpected sync ucall, got %lx",
-				(ulong)uc.args[1]);
+				(unsigned long)uc.args[1]);
 			sem_post(&vcpu_ready);
 			continue;
 		case UCALL_NONE:
@@ -339,7 +339,8 @@ static bool prepare_vm(struct vm_data *data, int nslots, u64 *maxslots,
 		if (slot == data->nslots)
 			npages += rempages;
 
-		gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot);
+		gpa = ____vm_phy_pages_alloc(data->vm, npages, guest_addr, slot,
+					     false, false);
 		TEST_ASSERT(gpa == guest_addr,
 			    "vm_phy_pages_alloc() failed");
 

tools/testing/selftests/kvm/s390/cmma_test.c

@@ -102,24 +102,17 @@ static void guest_dirty_test_data(void)
 
 static void create_main_memslot(struct kvm_vm *vm)
 {
-	int i;
-
 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, MAIN_PAGE_COUNT, 0);
-	/* set the array of memslots to zero like __vm_create does */
+	____vm_override_mem_region(vm, MEM_REGION_CODE, 0);
-	for (i = 0; i < NR_MEM_REGIONS; i++)
+	____vm_override_mem_region(vm, MEM_REGION_PT, 0);
-		vm->memslots[i] = 0;
+	____vm_override_mem_region(vm, MEM_REGION_DATA, 0);
 }
 
 static void create_test_memslot(struct kvm_vm *vm)
 {
-	vm_userspace_mem_region_add(vm,
+	vm_override_mem_region(vm, MEM_REGION_TEST_DATA, VM_MEM_SRC_ANONYMOUS,
-				    VM_MEM_SRC_ANONYMOUS,
+			       TEST_DATA_START_GFN << vm->page_shift,
-				    TEST_DATA_START_GFN << vm->page_shift,
+			       TEST_DATA_MEMSLOT, TEST_DATA_PAGE_COUNT);
-				    TEST_DATA_MEMSLOT,
-				    TEST_DATA_PAGE_COUNT,
-				    0
-				   );
-	vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT;
 }
 
 static void create_memslots(struct kvm_vm *vm)

tools/testing/selftests/kvm/s390/irq_routing.c

@@ -36,7 +36,7 @@ static void test(void)
 	};
 
 	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
-	mem = vm_phy_pages_alloc(vm, 2, 4096 * 42, 0);
+	mem = ____vm_phy_pages_alloc(vm, 2, 4096 * 42, 0, false, false);
 
 	routing = kvm_gsi_routing_create();
 	routing->nr = 1;

tools/testing/selftests/kvm/set_memory_region_test.c

@@ -116,15 +116,15 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread,
 
 	vm = vm_create_with_one_vcpu(vcpu, guest_code);
 
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
+	vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS_THP,
-				    MEM_REGION_GPA, MEM_REGION_SLOT,
+			       MEM_REGION_GPA, MEM_REGION_SLOT,
-				    MEM_REGION_SIZE / getpagesize(), 0);
+			       MEM_REGION_SIZE / getpagesize());
 
 	/*
 	 * Allocate and map two pages so that the GPA accessed by guest_code()
 	 * stays valid across the memslot move.
 	 */
-	gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT);
+	gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_TEST_EXTRA);
 	TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n");
 
 	virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2);

tools/testing/selftests/kvm/steal_time.c

@@ -76,12 +76,12 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 	WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i],
 				(void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE));
 
-	vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED);
+	vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (unsigned long)st_gva[i] | KVM_MSR_ENABLED);
 }
 
 static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
 {
-	struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
+	struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]);
 
 	ksft_print_msg("VCPU%d:\n", vcpu_idx);
 	ksft_print_msg("    steal:     %lld\n", st->steal);
@@ -105,7 +105,7 @@ static void check_steal_time_uapi(void)
 	vm = vm_create_with_one_vcpu(&vcpu, NULL);
 
 	ret = _vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME,
-			    (ulong)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK);
+			    (unsigned long)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK);
 	TEST_ASSERT(ret == 0, "Bad GPA didn't fail");
 
 	kvm_vm_free(vm);
@@ -154,7 +154,7 @@ static void guest_code(int cpu)
 
 	status = smccc(PV_TIME_ST, 0);
 	GUEST_ASSERT_NE(status, -1);
-	GUEST_ASSERT_EQ(status, (ulong)st_gva[cpu]);
+	GUEST_ASSERT_EQ(status, (unsigned long)st_gva[cpu]);
 
 	st = (struct st_time *)status;
 	GUEST_SYNC(0);
@@ -192,13 +192,13 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 	/* ST_GPA_BASE is identity mapped */
 	WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE));
 
-	st_ipa = (ulong)st_gva[i];
+	st_ipa = (unsigned long)st_gva[i];
 	vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
 }
 
 static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
 {
-	struct st_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
+	struct st_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]);
 
 	ksft_print_msg("VCPU%d:\n", vcpu_idx);
 	ksft_print_msg("    rev:     %d\n", st->rev);
@@ -222,11 +222,11 @@ static void check_steal_time_uapi(void)
 	vm = vm_create_steal_time(1, NULL, &vcpu);
 	vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev);
 
-	st_ipa = (ulong)ST_GPA_BASE | 1;
+	st_ipa = (unsigned long)ST_GPA_BASE | 1;
 	ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
 	TEST_ASSERT(ret == -1 && errno == EINVAL, "Bad IPA didn't report EINVAL");
 
-	st_ipa = (ulong)ST_GPA_BASE;
+	st_ipa = (unsigned long)ST_GPA_BASE;
 	vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
 
 	ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
@@ -315,7 +315,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 
 static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
 {
-	struct sta_struct *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
+	struct sta_struct *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]);
 	int i;
 
 	pr_info("VCPU%d:\n", vcpu_idx);
@@ -451,7 +451,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 
 static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
 {
-	struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
+	struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]);
 
 	ksft_print_msg("VCPU%d:\n", vcpu_idx);
 	ksft_print_msg("    steal:     %lld\n", st->steal);

tools/testing/selftests/kvm/x86/amx_test.c

@@ -325,7 +325,7 @@ int main(int argc, char *argv[])
 				vcpu_regs_get(vcpu, &regs2);
 				TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
 					    "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
-					    (ulong) regs2.rdi, (ulong) regs2.rsi);
+					    (unsigned long)regs2.rdi, (unsigned long)regs2.rsi);
 			}
 			break;
 		case UCALL_DONE:

tools/testing/selftests/kvm/x86/aperfmperf_test.c

@@ -70,8 +70,8 @@ static void l1_svm_code(struct svm_test_data *svm)
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT_EQ(load_vmcs(vmx), true);
+	load_vmcs(vmx);
 
 	prepare_vmcs(vmx, NULL);
 
@@ -80,10 +80,10 @@ static void l1_vmx_code(struct vmx_pages *vmx)
 	 * in the VMCS by prepare_vmcs()), as MSR exiting mandatory on Intel.
 	 */
 	vmwrite(CPU_BASED_VM_EXEC_CONTROL,
-		vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS);
+		vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS);
 
-	GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code));
+	vmwrite(GUEST_RIP, (u64)l2_guest_code);
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 }
 
 static void guest_code(void *nested_test_data)

tools/testing/selftests/kvm/x86/cpuid_test.c

@@ -128,7 +128,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage)
 		TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 			    uc.args[1] == stage + 1,
 			    "Stage %d: Unexpected register values vmexit, got %lx",
-			    stage + 1, (ulong)uc.args[1]);
+			    stage + 1, (unsigned long)uc.args[1]);
 		return;
 	case UCALL_DONE:
 		return;

tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c

@@ -55,11 +55,11 @@ static void guest_code(struct vmx_pages *vmx_pages,
 	enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist);
 	evmcs_enable();
 
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_evmcs(hv_pages));
+	load_evmcs(hv_pages);
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 
 	/* L2 exits via vmcall if test fails */
 	sync_with_host(2);

tools/testing/selftests/kvm/x86/feature_msrs_test.c

@@ -59,7 +59,7 @@ static void test_feature_msr(u32 msr)
 	 * but initializes the vCPU's revision ID to an arbitrary value.
 	 */
 	if (msr == MSR_IA32_UCODE_REV)
-		reset_value = host_cpu_is_intel ? 0x100000000ULL : 0x01000065;
+		reset_value = host_cpu_is_intel_compatible ? 0x100000000ULL : 0x01000065;
 
 	/*
 	 * For quirked MSRs, KVM's ABI is to initialize the vCPU's value to the

tools/testing/selftests/kvm/x86/fix_hypercall_test.c

@@ -48,7 +48,7 @@ static void guest_main(void)
 	const u8 *other_hypercall_insn;
 	u64 ret;
 
-	if (host_cpu_is_intel) {
+	if (host_cpu_is_intel_compatible) {
 		native_hypercall_insn = vmx_vmcall;
 		other_hypercall_insn  = svm_vmmcall;
 	} else if (host_cpu_is_amd_compatible) {

tools/testing/selftests/kvm/x86/hyperv_clock.c

@@ -248,7 +248,7 @@ int main(void)
 		TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 			    uc.args[1] == stage,
 			    "Stage %d: Unexpected register values vmexit, got %lx",
-			    stage, (ulong)uc.args[1]);
+			    stage, (unsigned long)uc.args[1]);
 
 		/* Reset kvmclock triggering TSC page update */
 		if (stage == 7 || stage == 8 || stage == 10) {

tools/testing/selftests/kvm/x86/hyperv_evmcs.c

@@ -23,7 +23,11 @@ static int ud_count;
 static void guest_ud_handler(struct ex_regs *regs)
 {
 	ud_count++;
-	regs->rip += 3; /* VMLAUNCH */
+	/*
+	 * VMLAUNCH insn can't be easily covered by KVM_ASM_SAFE framework but
+	 * luckily the instruction is always three bytes.
+	 */
+	regs->rip += 3;
 }
 
 static void guest_nmi_handler(struct ex_regs *regs)
@@ -89,24 +93,23 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages,
 	enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist);
 	evmcs_enable();
 
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
 	GUEST_SYNC(3);
-	GUEST_ASSERT(load_evmcs(hv_pages));
+	load_evmcs(hv_pages);
-	GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa);
+	/* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */
+	GUEST_ASSERT(vmptrst() == -1);
 
 	GUEST_SYNC(4);
-	GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa);
 
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	GUEST_SYNC(5);
-	GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa);
 	current_evmcs->revision_id = -1u;
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
 	current_evmcs->revision_id = EVMCS_VERSION;
 	GUEST_SYNC(6);
 
-	vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmreadz(PIN_BASED_VM_EXEC_CONTROL) |
+	vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmread(PIN_BASED_VM_EXEC_CONTROL) |
 		PIN_BASED_NMI_EXITING);
 
 	/* L2 TLB flush setup */
@@ -117,71 +120,79 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages,
 	current_vp_assist->nested_control.features.directhypercall = 1;
 	*(u32 *)(hv_pages->partition_assist) = 0;
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI);
-	GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR);
+	GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR);
-	GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa);
+	GUEST_ASSERT(vmptrst() == hv_pages->enlightened_vmcs_gpa);
 
 	/*
 	 * NMI forces L2->L1 exit, resuming L2 and hope that EVMCS is
 	 * up-to-date (RIP points where it should and not at the beginning
 	 * of l2_guest_code(). GUEST_SYNC(9) checks that.
 	 */
-	GUEST_ASSERT(!vmresume());
+	vmresume();
 
 	GUEST_SYNC(10);
 
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 	current_evmcs->guest_rip += 3; /* vmcall */
 
 	/* Intercept RDMSR 0xc0000100 */
-	vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) |
+	vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) |
 		CPU_BASED_USE_MSR_BITMAPS);
 	__set_bit(MSR_FS_BASE & 0x1fff, vmx_pages->msr + 0x400);
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
 	current_evmcs->guest_rip += 2; /* rdmsr */
 
 	/* Enable enlightened MSR bitmap */
 	current_evmcs->hv_enlightenments_control.msr_bitmap = 1;
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
 	current_evmcs->guest_rip += 2; /* rdmsr */
 
 	/* Intercept RDMSR 0xc0000101 without telling KVM about it */
 	__set_bit(MSR_GS_BASE & 0x1fff, vmx_pages->msr + 0x400);
 	/* Make sure HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP is set */
 	current_evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP;
-	GUEST_ASSERT(!vmresume());
+	vmresume();
 	/* Make sure we don't see EXIT_REASON_MSR_READ here so eMSR bitmap works */
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 	current_evmcs->guest_rip += 3; /* vmcall */
 
 	/* Now tell KVM we've changed MSR-Bitmap */
 	current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP;
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
 	current_evmcs->guest_rip += 2; /* rdmsr */
 
 	/*
 	 * L2 TLB flush test. First VMCALL should be handled directly by L0,
 	 * no VMCALL exit expected.
 	 */
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ);
 	current_evmcs->guest_rip += 2; /* rdmsr */
 	/* Enable synthetic vmexit */
 	*(u32 *)(hv_pages->partition_assist) = 1;
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH);
 
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 	GUEST_SYNC(11);
 
-	/* Try enlightened vmptrld with an incorrect GPA */
+	/* VMPTRLD instruction causes #UD after enlightened VMLAUNCH */
+	GUEST_ASSERT(__vmptrld(hv_pages->enlightened_vmcs_gpa) == UD_VECTOR);
+
+	/*
+	 * Try enlightened vmptrld with an incorrect GPA. GUEST_SYNC(12) signals
+	 * the host to enable guest_ud_handler() which cannot be enabled beforehand
+	 * to not override the default fixup handler from KVM_ASM_SAFE().
+	 */
+	GUEST_SYNC(12);
 	evmcs_vmptrld(0xdeadbeef, hv_pages->enlightened_vmcs);
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
 	GUEST_ASSERT(ud_count == 1);
 	GUEST_DONE();
 }
@@ -221,7 +232,7 @@ static struct kvm_vcpu *save_restore_vm(struct kvm_vm *vm,
 	vcpu_regs_get(vcpu, &regs2);
 	TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
 		    "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
-		    (ulong) regs2.rdi, (ulong) regs2.rsi);
+		    (unsigned long)regs2.rdi, (unsigned long)regs2.rsi);
 	return vcpu;
 }
 
@@ -253,7 +264,6 @@ int main(int argc, char *argv[])
 	vcpu_args_set(vcpu, 3, vmx_pages_gva, hv_pages_gva, addr_gva2gpa(vm, hcall_page));
 	vcpu_set_msr(vcpu, HV_X64_MSR_VP_INDEX, vcpu->id);
 
-	vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler);
 	vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler);
 
 	pr_info("Running L1 which uses EVMCS to run L2\n");
@@ -277,13 +287,13 @@ int main(int argc, char *argv[])
 		/* UCALL_SYNC is handled here.  */
 		TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 			    uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx",
-			    stage, (ulong)uc.args[1]);
+			    stage, (unsigned long)uc.args[1]);
 
 		vcpu = save_restore_vm(vm, vcpu);
 
 		/* Force immediate L2->L1 exit before resuming */
 		if (stage == 8) {
-			pr_info("Injecting NMI into L1 before L2 had a chance to run after restore\n");
+			pr_debug("Injecting NMI into L1 before L2 had a chance to run after restore\n");
 			inject_nmi(vcpu);
 		}
 
@@ -293,9 +303,14 @@ int main(int argc, char *argv[])
 		 * KVM_STATE_NESTED_EVMCS is not lost.
 		 */
 		if (stage == 9) {
-			pr_info("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n");
+			pr_debug("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n");
 			vcpu = save_restore_vm(vm, vcpu);
 		}
+
+		if (stage == 12) {
+			pr_debug("Trying enlightened VMLAUNCH with an invalid PTR\n");
+			vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler);
+		}
 	}
 
 done:

tools/testing/selftests/kvm/x86/hyperv_svm_test.c

@@ -186,7 +186,7 @@ int main(int argc, char *argv[])
 		/* UCALL_SYNC is handled here.  */
 		TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 			    uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx",
-			    stage, (ulong)uc.args[1]);
+			    stage, (unsigned long)uc.args[1]);
 
 	}
 

tools/testing/selftests/kvm/x86/kvm_buslock_test.c

@@ -42,13 +42,13 @@ static void l1_svm_code(struct svm_test_data *svm)
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT_EQ(load_vmcs(vmx), true);
+	load_vmcs(vmx);
 
 	prepare_vmcs(vmx, NULL);
 
-	GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code));
+	vmwrite(GUEST_RIP, (u64)l2_guest_code);
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 }
 
 static void guest_code(void *test_data)

tools/testing/selftests/kvm/x86/nested_close_kvm_test.c

@@ -30,13 +30,13 @@ static void l2_guest_code(void)
 
 static void l1_vmx_code(struct vmx_pages *vmx_pages)
 {
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	/* Prepare the VMCS for L2 execution. */
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 	GUEST_ASSERT(0);
 }
 

tools/testing/selftests/kvm/x86/nested_dirty_log_test.c

@@ -93,8 +93,8 @@ void l1_vmx_code(struct vmx_pages *vmx)
 	void *l2_rip;
 
 	GUEST_ASSERT(vmx->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT(load_vmcs(vmx));
+	load_vmcs(vmx);
 
 	if (vmx->eptp_gpa)
 		l2_rip = l2_guest_code_tdp_enabled;
@@ -104,9 +104,9 @@ void l1_vmx_code(struct vmx_pages *vmx)
 	prepare_vmcs(vmx, l2_rip);
 
 	GUEST_SYNC(TEST_SYNC_NO_FAULT);
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 	GUEST_SYNC(TEST_SYNC_NO_FAULT);
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL);
 	GUEST_DONE();
 }
 

tools/testing/selftests/kvm/x86/nested_emulation_test.c

@@ -66,15 +66,15 @@ static void guest_code(void *test_data)
 					   BIT_ULL(INTERCEPT_HLT);
 		vmcb->control.intercept_exceptions = 0;
 	} else {
-		GUEST_ASSERT(prepare_for_vmx_operation(test_data));
+		prepare_for_vmx_operation(test_data);
-		GUEST_ASSERT(load_vmcs(test_data));
+		load_vmcs(test_data);
 
 		prepare_vmcs(test_data, NULL);
-		GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0));
+		vmwrite(GUEST_IDTR_LIMIT, 0);
-		GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code));
+		vmwrite(GUEST_RIP, (u64)l2_guest_code);
-		GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, 0));
+		vmwrite(EXCEPTION_BITMAP, 0);
 
-		vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) |
+		vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) |
 						   CPU_BASED_PAUSE_EXITING |
 						   CPU_BASED_HLT_EXITING);
 	}
@@ -102,10 +102,13 @@ static void guest_code(void *test_data)
 			exit_insn_len = vmcb->control.next_rip - vmcb->save.rip;
 			GUEST_ASSERT_EQ(vmcb->save.rip, (u64)l2_instruction);
 		} else {
-			GUEST_ASSERT_EQ(i ? vmresume() : vmlaunch(), 0);
+			if (!i)
-			exit_reason = vmreadz(VM_EXIT_REASON);
+				vmlaunch();
-			exit_insn_len = vmreadz(VM_EXIT_INSTRUCTION_LEN);
+			else
-			GUEST_ASSERT_EQ(vmreadz(GUEST_RIP), (u64)l2_instruction);
+				vmresume();
+			exit_reason = vmread(VM_EXIT_REASON);
+			exit_insn_len = vmread(VM_EXIT_INSTRUCTION_LEN);
+			GUEST_ASSERT_EQ(vmread(GUEST_RIP), (u64)l2_instruction);
 		}
 
 		__GUEST_ASSERT(exit_reason == insn->exit_reason[f],

tools/testing/selftests/kvm/x86/nested_exceptions_test.c

@@ -38,6 +38,7 @@
  */
 #define GP_ERROR_CODE_AMD ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG)
 #define GP_ERROR_CODE_INTEL ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG | ERROR_CODE_EXT_FLAG)
+#define GP_ERROR_CODE_ZHAOXIN ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG)
 
 /*
  * Intel and AMD both shove '0' into the error code on #DF, regardless of what
@@ -110,43 +111,47 @@ static void l1_svm_code(struct svm_test_data *svm)
 
 static void vmx_run_l2(void *l2_code, int vector, u32 error_code)
 {
-	GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code));
+	vmwrite(GUEST_RIP, (u64)l2_code);
 
-	GUEST_ASSERT_EQ(vector == SS_VECTOR ? vmlaunch() : vmresume(), 0);
+	if (vector == SS_VECTOR)
+		vmlaunch();
+	else
+		vmresume();
 
 	if (vector == FAKE_TRIPLE_FAULT_VECTOR)
 		return;
 
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI);
-	GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), vector);
+	GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), vector);
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_ERROR_CODE), error_code);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_ERROR_CODE), error_code);
-	GUEST_ASSERT(!vmreadz(GUEST_INTERRUPTIBILITY_INFO));
+	GUEST_ASSERT(!vmread(GUEST_INTERRUPTIBILITY_INFO));
 }
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
+	prepare_for_vmx_operation(vmx);
 
-	GUEST_ASSERT_EQ(load_vmcs(vmx), true);
+	load_vmcs(vmx);
 
 	prepare_vmcs(vmx, NULL);
-	GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0);
+	vmwrite(GUEST_IDTR_LIMIT, 0);
 
 	/*
 	 * VMX disallows injecting an exception with error_code[31:16] != 0,
 	 * and hardware will never generate a VM-Exit with bits 31:16 set.
 	 * KVM should likewise truncate the "bad" userspace value.
 	 */
-	GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF), 0);
+	vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF);
 	vmx_run_l2(l2_ss_pending_test, SS_VECTOR, (u16)SS_ERROR_CODE);
-	vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_INTEL);
+	vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR,
+		   host_cpu_is_zhaoxin ? GP_ERROR_CODE_ZHAOXIN : GP_ERROR_CODE_INTEL);
 
-	GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF), 0);
+	vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF);
 	vmx_run_l2(l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE);
 
-	GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS), 0);
+	vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS);
 	vmx_run_l2(l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0);
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT);
 
 	GUEST_DONE();
 }

tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c

@@ -40,22 +40,22 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages)
 {
 	uintptr_t save_cr3;
 
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/* Try to run L2 with invalid CR3 and make sure it fails */
-	save_cr3 = vmreadz(GUEST_CR3);
+	save_cr3 = vmread(GUEST_CR3);
 	vmwrite(GUEST_CR3, -1ull);
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) ==
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) ==
 		     (EXIT_REASON_FAILED_VMENTRY | EXIT_REASON_INVALID_STATE));
 
 	/* Now restore CR3 and make sure L2 runs successfully */
 	vmwrite(GUEST_CR3, save_cr3);
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
 	GUEST_DONE();
 }

tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c

@@ -55,25 +55,25 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa,
 	u64 exit_qual;
 
 	GUEST_ASSERT(vmx->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT(load_vmcs(vmx));
+	load_vmcs(vmx);
 
 	prepare_vmcs(vmx, l2_entry);
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 
 	/* Verify we got an EPT violation exit */
-	__GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION,
+	__GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION,
 		       "Expected EPT violation (0x%x), got 0x%lx",
 		       EXIT_REASON_EPT_VIOLATION,
-		       vmreadz(VM_EXIT_REASON));
+		       vmread(VM_EXIT_REASON));
 
-	__GUEST_ASSERT(vmreadz(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa,
+	__GUEST_ASSERT(vmread(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa,
 		       "Expected guest_physical_address = 0x%lx, got 0x%lx",
 		       expected_fault_gpa,
-		       vmreadz(GUEST_PHYSICAL_ADDRESS));
+		       vmread(GUEST_PHYSICAL_ADDRESS));
 
-	exit_qual = vmreadz(EXIT_QUALIFICATION);
+	exit_qual = vmread(EXIT_QUALIFICATION);
 
 	/*
 	 * Note, EPT page table accesses are always read+write, e.g. so that

tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c

@@ -86,17 +86,17 @@ static void l1_guest_code(void *data)
 		struct vmx_pages *vmx_pages = data;
 		u32 control;
 
-		GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+		prepare_for_vmx_operation(vmx_pages);
-		GUEST_ASSERT(load_vmcs(vmx_pages));
+		load_vmcs(vmx_pages);
 
 		prepare_vmcs(vmx_pages, l2_guest_code);
-		control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
+		control = vmread(CPU_BASED_VM_EXEC_CONTROL);
 		control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING;
 		vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
 		vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE);
 
-		GUEST_ASSERT(!vmlaunch());
+		vmlaunch();
-		GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+		GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 	} else {
 		struct svm_test_data *svm = data;
 

tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c

@@ -1,14 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * vmx_nested_tsc_scaling_test
- *
  * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * This test case verifies that nested TSC scaling behaves as expected when
- * both L1 and L2 are scaled using different ratios. For this test we scale
+ * both L1 and L2 are scaled using different ratios.
- * L1 down and scale L2 up.
  */
-
+#include <linux/math64.h>
 #include <time.h>
 
 #include "kvm_util.h"
@@ -16,16 +13,11 @@
 #include "svm_util.h"
 #include "kselftest.h"
 
-/* L2 is scaled up (from L1's perspective) by this factor */
+/* L2's TSC multiplier and offset, relative to L1. */
-#define L2_SCALE_FACTOR 4ULL
+static u64 l2_multiplier;
-
+static u64 l2_offset;
-#define TSC_OFFSET_L2 ((u64)-33125236320908)
-#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48)
-
-enum { USLEEP, UCHECK_L1, UCHECK_L2 };
-#define GUEST_SLEEP(sec)         ucall(UCALL_SYNC, 2, USLEEP, sec)
-#define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq)
 
+enum ucall_cmd { UCALL_SLEEP, UCALL_CHECK_L1, UCALL_CHECK_L2 };
 
 /*
  * This function checks whether the "actual" TSC frequency of a guest matches
@@ -33,25 +25,20 @@ enum { USLEEP, UCHECK_L1, UCHECK_L2 };
  * measurements, a difference of 1% between the actual and the expected value
  * is tolerated.
  */
-static void compare_tsc_freq(u64 actual, u64 expected)
+static void host_check_tsc_freq(int level, u64 actual, u64 expected)
 {
 	u64 tolerance, thresh_low, thresh_high;
 
-	tolerance = expected / 100;
+	tolerance = max(expected / 100, (u64)1);
 	thresh_low = expected - tolerance;
 	thresh_high = expected + tolerance;
 
-	TEST_ASSERT(thresh_low < actual,
+	TEST_ASSERT(thresh_low <= actual && thresh_high >= actual,
-		"TSC freq is expected to be between %"PRIu64" and %"PRIu64
+		    "L%u TSC freq is '%lu', expected to be between %lu and %lu",
-		" but it actually is %"PRIu64,
+		    level, actual, thresh_low, thresh_high);
-		thresh_low, thresh_high, actual);
-	TEST_ASSERT(thresh_high > actual,
-		"TSC freq is expected to be between %"PRIu64" and %"PRIu64
-		" but it actually is %"PRIu64,
-		thresh_low, thresh_high, actual);
 }
 
-static void check_tsc_freq(int level)
+static void guest_check_tsc_freq(enum ucall_cmd check_level)
 {
 	u64 tsc_start, tsc_end, tsc_freq;
 
@@ -62,17 +49,17 @@ static void check_tsc_freq(int level)
 	 * be good enough for the purposes of this test.
 	 */
 	tsc_start = rdmsr(MSR_IA32_TSC);
-	GUEST_SLEEP(1);
+	GUEST_SYNC2(UCALL_SLEEP, 1);
 	tsc_end = rdmsr(MSR_IA32_TSC);
 
 	tsc_freq = tsc_end - tsc_start;
 
-	GUEST_CHECK(level, tsc_freq);
+	GUEST_SYNC2(check_level, tsc_freq);
 }
 
 static void l2_guest_code(void)
 {
-	check_tsc_freq(UCHECK_L2);
+	guest_check_tsc_freq(UCALL_CHECK_L2);
 
 	/* exit to L1 */
 	__asm__ __volatile__("vmcall");
@@ -81,19 +68,20 @@ static void l2_guest_code(void)
 static void l1_svm_code(struct svm_test_data *svm)
 {
 	/* check that L1's frequency looks alright before launching L2 */
-	check_tsc_freq(UCHECK_L1);
+	guest_check_tsc_freq(UCALL_CHECK_L1);
 
 	generic_svm_setup(svm, l2_guest_code);
+	svm->vmcb->control.tsc_offset = l2_offset;
 
 	/* enable TSC scaling for L2 */
-	wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32);
+	wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier);
 
 	/* launch L2 */
 	run_guest(svm->vmcb, svm->vmcb_gpa);
 	GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL);
 
 	/* check that L1's frequency still looks good */
-	check_tsc_freq(UCHECK_L1);
+	guest_check_tsc_freq(UCALL_CHECK_L1);
 
 	GUEST_DONE();
 }
@@ -103,33 +91,32 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages)
 	u32 control;
 
 	/* check that L1's frequency looks alright before launching L2 */
-	check_tsc_freq(UCHECK_L1);
+	guest_check_tsc_freq(UCALL_CHECK_L1);
 
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	/* prepare the VMCS for L2 execution */
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/* enable TSC offsetting and TSC scaling for L2 */
-	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
+	control = vmread(CPU_BASED_VM_EXEC_CONTROL);
 	control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING;
 	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
 
-	control = vmreadz(SECONDARY_VM_EXEC_CONTROL);
+	control = vmread(SECONDARY_VM_EXEC_CONTROL);
 	control |= SECONDARY_EXEC_TSC_SCALING;
 	vmwrite(SECONDARY_VM_EXEC_CONTROL, control);
 
-	vmwrite(TSC_OFFSET, TSC_OFFSET_L2);
+	vmwrite(TSC_OFFSET, l2_offset);
-	vmwrite(TSC_MULTIPLIER, TSC_MULTIPLIER_L2);
+	vmwrite(TSC_MULTIPLIER, l2_multiplier);
-	vmwrite(TSC_MULTIPLIER_HIGH, TSC_MULTIPLIER_L2 >> 32);
 
 	/* launch L2 */
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
 	/* check that L1's frequency still looks good */
-	check_tsc_freq(UCHECK_L1);
+	guest_check_tsc_freq(UCALL_CHECK_L1);
 
 	GUEST_DONE();
 }
@@ -142,44 +129,24 @@ static void l1_guest_code(void *data)
 		l1_svm_code(data);
 }
 
-int main(int argc, char *argv[])
+static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq,
+			     u64 __l2_multiplier)
 {
 	struct kvm_vcpu *vcpu;
 	struct kvm_vm *vm;
-	gva_t guest_gva = 0;
+	gva_t guest_gva;
 
-	u64 tsc_start, tsc_end;
+	printf("Testing L0 freq = %lu, L1 freq = %lu, L2 freq = %lu, L2 mult = 0x%lx\n",
-	u64 tsc_khz;
+	       l0_tsc_freq, l1_tsc_freq, l2_tsc_freq, __l2_multiplier);
-	u64 l1_scale_factor;
-	u64 l0_tsc_freq = 0;
-	u64 l1_tsc_freq = 0;
-	u64 l2_tsc_freq = 0;
-
-	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
-		     kvm_cpu_has(X86_FEATURE_SVM));
-	TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL));
-	TEST_REQUIRE(sys_clocksource_is_based_on_tsc());
-
-	/*
-	 * We set L1's scale factor to be a random number from 2 to 10.
-	 * Ideally we would do the same for L2's factor but that one is
-	 * referenced by both main() and l1_guest_code() and using a global
-	 * variable does not work.
-	 */
-	srand(time(NULL));
-	l1_scale_factor = (rand() % 9) + 2;
-	printf("L1's scale down factor is: %"PRIu64"\n", l1_scale_factor);
-	printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR);
-
-	tsc_start = rdtsc();
-	sleep(1);
-	tsc_end = rdtsc();
-
-	l0_tsc_freq = tsc_end - tsc_start;
-	printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq);
 
 	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
 
+	l2_offset = kvm_random_u64(&kvm_rng);
+	sync_global_to_guest(vm, l2_offset);
+
+	l2_multiplier = __l2_multiplier;
+	sync_global_to_guest(vm, l2_multiplier);
+
 	if (kvm_cpu_has(X86_FEATURE_VMX))
 		vcpu_alloc_vmx(vm, &guest_gva);
 	else
@@ -187,11 +154,7 @@ int main(int argc, char *argv[])
 
 	vcpu_args_set(vcpu, 1, guest_gva);
 
-	tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL);
+	vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)(l1_tsc_freq / 1000));
-	TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed");
-
-	/* scale down L1's TSC frequency */
-	vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (tsc_khz / l1_scale_factor));
 
 	for (;;) {
 		struct ucall uc;
@@ -204,24 +167,16 @@ int main(int argc, char *argv[])
 			REPORT_GUEST_ASSERT(uc);
 		case UCALL_SYNC:
 			switch (uc.args[0]) {
-			case USLEEP:
+			case UCALL_SLEEP:
 				sleep(uc.args[1]);
 				break;
-			case UCHECK_L1:
+			case UCALL_CHECK_L1:
-				l1_tsc_freq = uc.args[1];
+				printf("L1's observed TSC frequency: %lu\n", uc.args[1]);
-				printf("L1's TSC frequency is around: %"PRIu64
+				host_check_tsc_freq(1, uc.args[1], l1_tsc_freq);
-				       "\n", l1_tsc_freq);
-
-				compare_tsc_freq(l1_tsc_freq,
-						 l0_tsc_freq / l1_scale_factor);
 				break;
-			case UCHECK_L2:
+			case UCALL_CHECK_L2:
-				l2_tsc_freq = uc.args[1];
+				printf("L2's observed TSC frequency: %lu\n", uc.args[1]);
-				printf("L2's TSC frequency is around: %"PRIu64
+				host_check_tsc_freq(2, uc.args[1], l2_tsc_freq);
-				       "\n", l2_tsc_freq);
-
-				compare_tsc_freq(l2_tsc_freq,
-						 l1_tsc_freq * L2_SCALE_FACTOR);
 				break;
 			}
 			break;
@@ -234,5 +189,108 @@ int main(int argc, char *argv[])
 
 done:
 	kvm_vm_free(vm);
+}
+
+int main(int argc, char *argv[])
+{
+	u64 min_freq, min_multiplier, max_freq, max_multiplier, l1_max_freq, l1_min_freq;
+	u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale;
+	u8 frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32;
+	struct kvm_vm *vm;
+
+	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
+		     kvm_cpu_has(X86_FEATURE_SVM));
+	TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL));
+	TEST_REQUIRE(sys_clocksource_is_based_on_tsc());
+
+	/*
+	 * Create a dummy VM to get KVM's default TSC frequency.  All CPUs that
+	 * support TSC scaling should have a constant TSC, i.e. there's no need
+	 * to calibrate the "real" TSC.  But do sanity check that the observed
+	 * TSC is within range of KVM's reported TSC frequency.
+	 */
+	vm = vm_create_barebones();
+	l0_tsc_freq = (u64)__vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL) * 1000;
+	TEST_ASSERT(l0_tsc_freq, "vcpu ioctl KVM_GET_TSC_KHZ failed");
+	kvm_vm_free(vm);
+
+	printf("L0 TSC frequency is: %lu\n", l0_tsc_freq);
+
+	tsc_start = rdtsc();
+	sleep(1);
+	tsc_end = rdtsc();
+	host_check_tsc_freq(0, tsc_end - tsc_start, l0_tsc_freq);
+
+	/* Sanity check the frequency reported by KVM_GET_TSC_KHZ. */
+	test_tsc_scaling(l0_tsc_freq, l0_tsc_freq, l0_tsc_freq, BIT_ULL(frac_bits));
+
+	/*
+	 * Scale L1 and L2 up and down at random factors from 2 to 10.  Current
+	 * CPUs have two full orders of magnitude of "breathing room" before an
+	 * ultrafast L0 TSC frequency multiplied by 10x will encounter KVM's
+	 * signed 32-bit limit on L1's frequency, and it's highly unlikely a
+	 * CPU that supports TSC scaling will show up running at 100MHz, i.e.
+	 * underflowing KVM's minimum 1KHz frequency is extremely unlikely.
+	 *
+	 * For L2, the frequency is limited only by what hardware can support,
+	 * not by KVM's limits.  SVM provides 8 bits of integer scale up, and
+	 * 32 bits of fractional scale down, i.e. can scale up 255x and down a
+	 * comical amount, so scaling up 100x and down 1/100 is well within
+	 * hardware's capabilities (VMX provides 16 bits of "up" and 48 bits of
+	 * "down").
+	 */
+	l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2;
+	l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2;
+
+	test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale,
+			 mul_u64_u64_div64(l0_tsc_freq, l2_scale, l1_scale),
+			 (l2_scale << frac_bits));
+
+	test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale,
+			 mul_u64_u64_div64(l0_tsc_freq, l1_scale, l2_scale),
+			 (1ull << frac_bits) / l2_scale);
+
+	test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale,
+			 l0_tsc_freq / l1_scale / l2_scale,
+			 (1ull << frac_bits) / l2_scale);
+
+	test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale,
+			 l0_tsc_freq * l1_scale * l2_scale,
+			 (l2_scale << frac_bits));
+
+	/*
+	 * Test that KVM saturates L2's frequency on both ends if the resulting
+	 * L2 TSC frequency would be below or above what hardware can support.
+	 * Because L2 = L0 * (L1_mult >> frac) * (L2_mult >> frac) needs to be
+	 * distilled down to a single multiplier, very small/large multipliers
+	 * will underflow/overflow the minimum/maximum multiplier supported by
+	 * hardware when L1 and L2 multipliers are combined.  KVM's behavior is
+	 * saturate on {under,over}flow, i.e. to run at the min/max frequency.
+	 *
+	 * Note, userspace can only program L1's frequency in KHz, i.e. can't
+	 * specify an exact multiplier.  As a result, the minimum and maximum
+	 * frequencies are different for L1 vs L2, because L1 is constrained by
+	 * hardware *and* KVM, whereas L2 is constrained only by hardware.
+	 */
+	min_multiplier = 1;
+	min_freq = mul_u64_u64_div64(l0_tsc_freq, min_multiplier, BIT_ULL(frac_bits));
+	min_freq = max(min_freq, (u64)1);
+	l1_min_freq = max(min_freq, (u64)1 * 1000);
+	test_tsc_scaling(l0_tsc_freq, l1_min_freq, min_freq, 1);
+
+	/*
+	 * SVM takes a 40-bit value (right shifted by 32), while VMX takes a
+	 * 64-bit value (right shifted by 48).  mul_u64_u64_shr() isn't (yet)
+	 * available in selftests, but mul_u64_u64_div64() does nicely since,
+	 * albeit more slowly (performance is obviously not a concern).  Note,
+	 * because KVM_GET_TSC_KHZ returns a signed 32-bit integer, KVM limits
+	 * KVM_SET_TSC_KHZ to INT_MAX, even though hardware (both SVM and VMX)
+	 * supports much higher frequencies.
+	 */
+	max_multiplier = kvm_cpu_has(X86_FEATURE_VMX) ? -1ull : GENMASK_U64(39, 0);
+	max_freq = mul_u64_u64_div64(l0_tsc_freq, max_multiplier, BIT_ULL(frac_bits));
+	l1_max_freq = min(max_freq, (u64)INT32_MAX * 1000);
+	test_tsc_scaling(l0_tsc_freq, l1_max_freq, max_freq, max_multiplier);
+
 	return 0;
 }

tools/testing/selftests/kvm/x86/nested_x2apic_test.c

@@ -61,57 +61,55 @@ static void l1_vmx_code(struct vmx_pages *vmx, struct hyperv_test_pages *hv_page
 		evmcs_enable();
 	}
 
-	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
+	prepare_for_vmx_operation(vmx);
 
 	if (hv_pages) {
-		GUEST_ASSERT(load_evmcs(hv_pages));
+		load_evmcs(hv_pages);
 		current_evmcs->hv_enlightenments_control.msr_bitmap = 1;
 	} else {
-		GUEST_ASSERT(load_vmcs(vmx));
+		load_vmcs(vmx);
 	}
 
 	prepare_vmcs(vmx, NULL);
-	GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0);
+	vmwrite(GUEST_RIP, (unsigned long)l2_guest_code);
 
-	control = vmreadz(PIN_BASED_VM_EXEC_CONTROL);
+	control = vmread(PIN_BASED_VM_EXEC_CONTROL);
 	control |= PIN_BASED_EXT_INTR_MASK;
 	vmwrite(PIN_BASED_VM_EXEC_CONTROL, control);
 
-	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
+	control = vmread(CPU_BASED_VM_EXEC_CONTROL);
 	control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_TPR_SHADOW;
-	GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0);
+	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
 
-	control = vmreadz(SECONDARY_VM_EXEC_CONTROL);
+	control = vmread(SECONDARY_VM_EXEC_CONTROL);
 	control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
 		   SECONDARY_EXEC_APIC_REGISTER_VIRT |
 		   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
 	control &= (rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32);
-	GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0);
+	vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0;
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID);
-	GUEST_ASSERT_EQ(vmwrite(GUEST_RIP,
+	vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN));
-			vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0);
 }
 
 static void l1_vmx_code_part2(void)
 {
 	u64 control;
 
-	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
+	control = vmread(CPU_BASED_VM_EXEC_CONTROL);
 	control &= ~CPU_BASED_TPR_SHADOW;
-	GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0);
+	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
 
-	control = vmreadz(SECONDARY_VM_EXEC_CONTROL);
+	control = vmread(SECONDARY_VM_EXEC_CONTROL);
 	control &= ~(SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
 			SECONDARY_EXEC_APIC_REGISTER_VIRT |
 			SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
-	GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0);
+	vmwrite(SECONDARY_VM_EXEC_CONTROL, control);
 
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID);
+	GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID);
-	GUEST_ASSERT_EQ(vmwrite(GUEST_RIP,
+	vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN));
-			vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0);
 }
 
 static void l1_test_x2apic_intercepts(void)

tools/testing/selftests/kvm/x86/nx_huge_pages_test.c

@@ -120,7 +120,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages,
 		} else {
 			TEST_ASSERT(r == -1 && errno == EPERM,
 				    "This process should not have permission to disable NX huge pages");
-			return;
+			goto done;
 		}
 	}
 
@@ -213,6 +213,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages,
 	check_2m_page_count(vm, disable_nx_huge_pages ? 3 : 2);
 	check_split_count(vm, 0);
 
+done:
 	kvm_vm_free(vm);
 }
 

tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c

@@ -98,17 +98,17 @@ static void l1_svm_code(struct svm_test_data *svm)
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT(load_vmcs(vmx));
+	load_vmcs(vmx);
 	prepare_vmcs(vmx, guest_access_memory);
 
-	GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR)));
+	vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR));
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 	while (1) {
-		GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI);
+		GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI);
-		GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR);
+		GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR);
-		GUEST_ASSERT(!vmresume());
+		vmresume();
 	}
 }
 

tools/testing/selftests/kvm/x86/set_boot_cpu_id.c

@@ -69,7 +69,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu)
 			TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 					uc.args[1] == stage + 1,
 					"Stage %d: Unexpected register values vmexit, got %lx",
-					stage + 1, (ulong)uc.args[1]);
+					stage + 1, (unsigned long)uc.args[1]);
 			test_set_bsp_busy(vcpu, "while running vm");
 			break;
 		case UCALL_DONE:

tools/testing/selftests/kvm/x86/sev_dbg_test.c

@@ -14,9 +14,11 @@ static u8 *data;
 static u8 src[BUFFER_SIZE] __aligned(PAGE_SIZE);
 static u8 dst[BUFFER_SIZE] __aligned(PAGE_SIZE);
 
-static void validate_dst(int i, int nr_bytes, u8 pattern)
+static void validate_dst(int offset, int nr_bytes, u8 pattern)
 {
-	for ( ; i < nr_bytes; i++)
+	int i;
+
+	for (i = offset; i < offset + nr_bytes; i++)
 		TEST_ASSERT(dst[i] == pattern,
 			    "Expected 0x%x at byte %u, got 0x%x",
 			    pattern, i, dst[i]);

tools/testing/selftests/kvm/x86/sev_smoke_test.c

@@ -178,7 +178,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy)
 		case UCALL_SYNC:
 			continue;
 		case UCALL_DONE:
-			return;
+			goto done;
 		case UCALL_ABORT:
 			REPORT_GUEST_ASSERT(uc);
 		default:
@@ -187,6 +187,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy)
 		}
 	}
 
+done:
 	kvm_vm_free(vm);
 }
 

tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c

@@ -62,11 +62,12 @@ int main(int argc, char *argv[])
 	TEST_ASSERT(rc, "KVM_CAP_EXIT_ON_EMULATION_FAILURE is unavailable");
 	vm_enable_cap(vm, KVM_CAP_EXIT_ON_EMULATION_FAILURE, 1);
 
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
+	vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS,
-				    MEM_REGION_GPA, MEM_REGION_SLOT,
+			       MEM_REGION_GPA, MEM_REGION_SLOT,
-				    MEM_REGION_SIZE / PAGE_SIZE, 0);
+			       MEM_REGION_SIZE / PAGE_SIZE);
+
 	gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE,
-				 MEM_REGION_GPA, MEM_REGION_SLOT);
+				 MEM_REGION_TEST_EXTRA);
 	TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc");
 	virt_map(vm, MEM_REGION_GVA, MEM_REGION_GPA, 1);
 	hva = addr_gpa2hva(vm, MEM_REGION_GPA);

tools/testing/selftests/kvm/x86/smm_test.c

@@ -78,8 +78,8 @@ static void guest_code(void *arg)
 		if (this_cpu_has(X86_FEATURE_SVM)) {
 			generic_svm_setup(svm, l2_guest_code);
 		} else {
-			GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+			prepare_for_vmx_operation(vmx_pages);
-			GUEST_ASSERT(load_vmcs(vmx_pages));
+			load_vmcs(vmx_pages);
 			prepare_vmcs(vmx_pages, l2_guest_code);
 		}
 

tools/testing/selftests/kvm/x86/state_test.c

@@ -59,13 +59,13 @@ void vmx_l2_guest_code(void)
 	vmcall();
 
 	/* L1 has now set up a shadow VMCS for us.  */
-	GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee);
+	GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee);
 	GUEST_SYNC(10);
-	GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee);
+	GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee);
-	GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0fffee));
+	vmwrite(GUEST_RIP, 0xc0fffee);
 	GUEST_SYNC(11);
-	GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0fffee);
+	GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0fffee);
-	GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0ffffee));
+	vmwrite(GUEST_RIP, 0xc0ffffee);
 	GUEST_SYNC(12);
 
 	/* Done, exit to L1 and never come back.  */
@@ -75,61 +75,61 @@ void vmx_l2_guest_code(void)
 static void vmx_l1_guest_code(struct vmx_pages *vmx_pages)
 {
 	GUEST_ASSERT(vmx_pages->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
 	GUEST_SYNC(3);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
-	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
+	GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa);
 
 	GUEST_SYNC(4);
-	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
+	GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa);
 
 	prepare_vmcs(vmx_pages, vmx_l2_guest_code);
 
 	GUEST_SYNC(5);
-	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
+	GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa);
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
+	GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa);
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
 	/* Check that the launched state is preserved.  */
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
 
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
 	GUEST_SYNC(7);
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
-	vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + 3);
+	vmwrite(GUEST_RIP, vmread(GUEST_RIP) + 3);
 
 	vmwrite(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS);
 	vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa);
 
-	GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa));
+	vmptrld(vmx_pages->shadow_vmcs_gpa);
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
 	GUEST_SYNC(8);
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
-	GUEST_ASSERT(vmresume());
+	GUEST_ASSERT(__vmresume());
 
 	vmwrite(GUEST_RIP, 0xc0ffee);
 	GUEST_SYNC(9);
-	GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee);
+	GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee);
 
-	GUEST_ASSERT(!vmptrld(vmx_pages->vmcs_gpa));
+	vmptrld(vmx_pages->vmcs_gpa);
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
-	GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa));
+	vmptrld(vmx_pages->shadow_vmcs_gpa);
-	GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee);
+	GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee);
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
-	GUEST_ASSERT(vmresume());
+	GUEST_ASSERT(__vmresume());
 	GUEST_SYNC(13);
-	GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee);
+	GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee);
-	GUEST_ASSERT(vmlaunch());
+	GUEST_ASSERT(__vmlaunch());
-	GUEST_ASSERT(vmresume());
+	GUEST_ASSERT(__vmresume());
 }
 
 static void __attribute__((__flatten__)) guest_code(void *arg)
@@ -296,7 +296,7 @@ int main(int argc, char *argv[])
 		/* UCALL_SYNC is handled here.  */
 		TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 			    uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx",
-			    stage, (ulong)uc.args[1]);
+			    stage, (unsigned long)uc.args[1]);
 
 		state = vcpu_save_state(vcpu);
 		memset(&regs1, 0, sizeof(regs1));
@@ -343,7 +343,7 @@ int main(int argc, char *argv[])
 		vcpu_regs_get(vcpu, &regs2);
 		TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
 			    "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
-			    (ulong) regs2.rdi, (ulong) regs2.rsi);
+			    (unsigned long)regs2.rdi, (unsigned long)regs2.rsi);
 	}
 
 done:

tools/testing/selftests/kvm/x86/triple_fault_event_test.c

@@ -25,14 +25,14 @@ void l1_guest_code_vmx(struct vmx_pages *vmx)
 {
 
 	GUEST_ASSERT(vmx->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
+	prepare_for_vmx_operation(vmx);
-	GUEST_ASSERT(load_vmcs(vmx));
+	load_vmcs(vmx);
 
 	prepare_vmcs(vmx, l2_guest_code);
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 	/* L2 should triple fault after a triple fault event injected. */
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT);
 	GUEST_DONE();
 }
 

tools/testing/selftests/kvm/x86/tsc_msrs_test.c

@@ -78,7 +78,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage)
 		else
 			ksft_test_result_fail(
 				"stage %d: Unexpected register values vmexit, got %lx",
-				stage + 1, (ulong)uc.args[1]);
+				stage + 1, (unsigned long)uc.args[1]);
 		return;
 	case UCALL_DONE:
 		ksft_test_result_pass("stage %d passed\n", stage + 1);

tools/testing/selftests/kvm/x86/vmx_apic_access_test.c

@@ -38,30 +38,30 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa)
 {
 	u32 control;
 
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	/* Prepare the VMCS for L2 execution. */
 	prepare_vmcs(vmx_pages, l2_guest_code);
-	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
+	control = vmread(CPU_BASED_VM_EXEC_CONTROL);
 	control |= CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
 	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
-	control = vmreadz(SECONDARY_VM_EXEC_CONTROL);
+	control = vmread(SECONDARY_VM_EXEC_CONTROL);
 	control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
 	vmwrite(SECONDARY_VM_EXEC_CONTROL, control);
 	vmwrite(APIC_ACCESS_ADDR, vmx_pages->apic_access_gpa);
 
 	/* Try to launch L2 with the memory-backed APIC-access address. */
-	GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR));
+	GUEST_SYNC(vmread(APIC_ACCESS_ADDR));
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
 	vmwrite(APIC_ACCESS_ADDR, high_gpa);
 
 	/* Try to resume L2 with the unbacked APIC-access address. */
-	GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR));
+	GUEST_SYNC(vmread(APIC_ACCESS_ADDR));
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 
 	GUEST_DONE();
 }
@@ -77,7 +77,7 @@ int main(int argc, char *argv[])
 	struct kvm_vcpu *vcpu;
 	struct kvm_vm *vm;
 
-	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX));
+	TEST_REQUIRE(kvm_cpu_has_vmx_virtualize_apic_accesses());
 
 	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
 

tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c

@@ -33,12 +33,12 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 {
 	u32 control;
 
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	/* Prepare the VMCS for L2 execution. */
 	prepare_vmcs(vmx_pages, l2_guest_code);
-	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
+	control = vmread(CPU_BASED_VM_EXEC_CONTROL);
 	control |= CPU_BASED_USE_MSR_BITMAPS;
 	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
 
@@ -62,9 +62,9 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 	 * Run L2 to switch to x2APIC mode, which in turn will uninhibit APICv,
 	 * as KVM should force the APIC ID back to its default.
 	 */
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
-	vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN));
+	vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN));
 	GUEST_ASSERT(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD);
 
 	/*
@@ -90,8 +90,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 	 * handles the x2APIC => xAPIC transition and inhibits APICv while L2
 	 * is active.
 	 */
-	GUEST_ASSERT(!vmresume());
+	vmresume();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
 	GUEST_ASSERT(!(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD));
 
 	xapic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_DM_FIXED | GOOD_IPI_VECTOR);

tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c

@@ -109,7 +109,7 @@ int main(int argc, char *argv[])
 	struct kvm_vcpu *vcpu;
 	struct kvm_vm *vm;
 
-	TEST_REQUIRE(host_cpu_is_intel);
+	TEST_REQUIRE(host_cpu_is_intel_compatible);
 	TEST_REQUIRE(!kvm_is_unrestricted_guest_enabled());
 
 	vm = vm_create_with_one_vcpu(&vcpu, guest_code);

tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c

@@ -41,8 +41,8 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct vmx_pages *vmx_pages)
 {
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	/* Prepare the VMCS for L2 execution. */
 	prepare_vmcs(vmx_pages, l2_guest_code);
@@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 	 * library hasn't enabled it.  Because KVM selftests jump directly to
 	 * 64-bit mode, unrestricted guest support isn't required.
 	 */
-	GUEST_ASSERT(!(vmreadz(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) ||
+	GUEST_ASSERT(!(vmread(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) ||
-		     !(vmreadz(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST));
+		     !(vmread(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST));
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 
 	/* L2 should triple fault after main() stuffs invalid guest state. */
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT);
 	GUEST_DONE();
 }
 

tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c

@@ -37,8 +37,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 	GUEST_ASSERT(rdmsr(MSR_GS_BASE) == LA57_GS_BASE);
 
 	GUEST_ASSERT(vmx_pages->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
 
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
@@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 	pml4_pa = pml5[0] & PHYSICAL_PAGE_MASK;
 	vmwrite(GUEST_CR3, pml4_pa);
 
-	guest_cr4 = vmreadz(GUEST_CR4);
+	guest_cr4 = vmread(GUEST_CR4);
 	guest_cr4 &= ~X86_CR4_LA57;
 	vmwrite(GUEST_CR4, guest_cr4);
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
 
-	exit_reason = vmreadz(VM_EXIT_REASON);
+	exit_reason = vmread(VM_EXIT_REASON);
 	GUEST_ASSERT(exit_reason == EXIT_REASON_VMCALL);
 }
 
@@ -109,7 +109,7 @@ int main(int argc, char *argv[])
 		}
 
 		TEST_ASSERT(uc.args[1] == stage,
-			    "Expected stage %d, got stage %lu", stage, (ulong)uc.args[1]);
+			    "Expected stage %d, got stage %lu", stage, (unsigned long)uc.args[1]);
 		if (stage == 1) {
 			pr_info("L2 is active; performing save/restore.\n");
 			state = vcpu_save_state(vcpu);

tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c

@@ -71,9 +71,9 @@ void l1_guest_code(struct vmx_pages *vmx_pages)
 	u64 l1_tsc_deadline, l2_tsc_deadline;
 
 	GUEST_ASSERT(vmx_pages->vmcs_gpa);
-	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
+	prepare_for_vmx_operation(vmx_pages);
-	GUEST_ASSERT(load_vmcs(vmx_pages));
+	load_vmcs(vmx_pages);
-	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
+	GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa);
 
 	prepare_vmcs(vmx_pages, l2_guest_code);
 
@@ -90,19 +90,17 @@ void l1_guest_code(struct vmx_pages *vmx_pages)
 	    !(ctrl_exit_rev.clr & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))
 		return;
 
-	GUEST_ASSERT(!vmlaunch());
+	vmlaunch();
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
-	vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN));
+	vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN));
 
 	/*
 	 * Turn on PIN control and resume the guest
 	 */
-	GUEST_ASSERT(!vmwrite(PIN_BASED_VM_EXEC_CONTROL,
+	vmwrite(PIN_BASED_VM_EXEC_CONTROL,
-			      vmreadz(PIN_BASED_VM_EXEC_CONTROL) |
+		vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_VMX_PREEMPTION_TIMER);
-			      PIN_BASED_VMX_PREEMPTION_TIMER));
 
-	GUEST_ASSERT(!vmwrite(VMX_PREEMPTION_TIMER_VALUE,
+	vmwrite(VMX_PREEMPTION_TIMER_VALUE, PREEMPTION_TIMER_VALUE);
-			      PREEMPTION_TIMER_VALUE));
 
 	vmx_pt_rate = rdmsr(MSR_IA32_VMX_MISC) & 0x1F;
 
@@ -110,7 +108,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages)
 
 	l1_vmx_pt_start = (rdtsc() >> vmx_pt_rate) << vmx_pt_rate;
 
-	GUEST_ASSERT(!vmresume());
+	vmresume();
 
 	l1_vmx_pt_finish = rdtsc();
 
@@ -123,7 +121,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages)
 	/*
 	 * Ensure the exit from L2 is due to preemption timer expiry
 	 */
-	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER);
+	GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER);
 
 	l1_tsc_deadline = l1_vmx_pt_start +
 		(PREEMPTION_TIMER_VALUE << vmx_pt_rate);
@@ -193,7 +191,7 @@ int main(int argc, char *argv[])
 		/* UCALL_SYNC is handled here.  */
 		TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
 			    uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx",
-			    stage, (ulong)uc.args[1]);
+			    stage, (unsigned long)uc.args[1]);
 		/*
 		 * If this stage 2 then we should verify the vmx pt expiry
 		 * is as expected.
@@ -234,7 +232,7 @@ int main(int argc, char *argv[])
 		vcpu_regs_get(vcpu, &regs2);
 		TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
 			    "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
-			    (ulong) regs2.rdi, (ulong) regs2.rsi);
+			    (unsigned long)regs2.rdi, (unsigned long)regs2.rsi);
 	}
 
 done:

tools/testing/selftests/kvm/x86/xapic_ipi_test.c

@@ -233,13 +233,10 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
 {
 	long pages_not_moved;
 	unsigned long nodemask = 0;
-	unsigned long nodemasks[sizeof(nodemask) * 8];
 	int nodes = 0;
 	time_t start_time, last_update, now;
 	time_t interval_secs = 1;
-	int i;
 	int from, to;
-	unsigned long bit;
 	u64 hlt_count;
 	u64 wake_count;
 	u64 ipis_sent;
@@ -247,32 +244,19 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
 	fprintf(stderr, "Calling migrate_pages every %d microseconds\n",
 		delay_usecs);
 
-	/* Get set of first 64 numa nodes available */
+	nodes = kvm_get_numa_memory_nodes(&nodemask);
-	kvm_get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8,
+	TEST_ASSERT(nodes > 1,
-			  0, MPOL_F_MEMS_ALLOWED);
+		    "NUMA nodes disappeared?  nodemask = 0x%lx", nodemask);
 
 	fprintf(stderr, "Numa nodes found amongst first %lu possible nodes "
 		"(each 1-bit indicates node is present): %#lx\n",
-		sizeof(nodemask) * 8, nodemask);
+		BITS_PER_TYPE(nodemask), nodemask);
-
-	/* Init array of masks containing a single-bit in each, one for each
-	 * available node. migrate_pages called below requires specifying nodes
-	 * as bit masks.
-	 */
-	for (i = 0, bit = 1; i < sizeof(nodemask) * 8; i++, bit <<= 1) {
-		if (nodemask & bit) {
-			nodemasks[nodes] = nodemask & bit;
-			nodes++;
-		}
-	}
-
-	TEST_ASSERT(nodes > 1,
-		    "Did not find at least 2 numa nodes. Can't do migration");
 
 	fprintf(stderr, "Migrating amongst %d nodes found\n", nodes);
 
-	from = 0;
+	from = kvm_get_next_numa_node(nodemask, -1);
-	to = 1;
+	to = kvm_get_next_numa_node(nodemask, from);
+
 	start_time = time(NULL);
 	last_update = start_time;
 
@@ -281,6 +265,9 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
 	wake_count = data->wake_count;
 
 	while ((int)(time(NULL) - start_time) < run_secs) {
+		unsigned long from_mask = BIT(from);
+		unsigned long to_mask = BIT(to);
+
 		data->migrations_attempted++;
 
 		/*
@@ -291,9 +278,8 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
 		 * KVM_CREATE_VCPU ioctl. If that assumption ever changes this
 		 * test may break or give a false positive signal.
 		 */
-		pages_not_moved = migrate_pages(0, sizeof(nodemasks[from]),
+		pages_not_moved = migrate_pages(0, MAXNODE_FOR_MASK(from_mask),
-						&nodemasks[from],
+						&from_mask, &to_mask);
-						&nodemasks[to]);
 		if (pages_not_moved < 0)
 			fprintf(stderr,
 				"migrate_pages failed, errno=%d\n", errno);
@@ -305,9 +291,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
 			data->migrations_completed++;
 
 		from = to;
-		to++;
+		to = kvm_get_next_numa_node(nodemask, from);
-		if (to == nodes)
-			to = 0;
 
 		now = time(NULL);
 		if (((now - start_time) % interval_secs == 0) &&
@@ -366,26 +350,17 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs,
 	}
 }
 
-int main(int argc, char *argv[])
+static void test_xapic_ipi(int run_secs, int delay_usecs, bool migrate)
 {
 	int wait_secs;
 	const int max_halter_wait = 10;
-	int run_secs = 0;
-	int delay_usecs = 0;
 	struct test_data_page *data;
 	gva_t test_data_page_gva;
-	bool migrate = false;
 	pthread_t threads[2];
 	struct thread_params params[2];
 	struct kvm_vm *vm;
 	u64 *pipis_rcvd;
 
-	get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs);
-	if (run_secs <= 0)
-		run_secs = DEFAULT_RUN_SECS;
-	if (delay_usecs <= 0)
-		delay_usecs = DEFAULT_DELAY_USECS;
-
 	vm = vm_create_with_one_vcpu(&params[0].vcpu, halter_guest_code);
 
 	vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler);
@@ -473,5 +448,30 @@ int main(int argc, char *argv[])
 
 	kvm_vm_free(vm);
 
+}
+
+int main(int argc, char *argv[])
+{
+	bool force_migrate = false;
+	unsigned long nodemask;
+	int run_secs = 0;
+	int delay_usecs = 0;
+
+	get_cmdline_args(argc, argv, &run_secs, &force_migrate, &delay_usecs);
+	if (run_secs <= 0)
+		run_secs = DEFAULT_RUN_SECS;
+	if (delay_usecs <= 0)
+		delay_usecs = DEFAULT_DELAY_USECS;
+
+	if (!force_migrate)
+		test_xapic_ipi(run_secs, delay_usecs, false);
+
+	if (kvm_get_numa_memory_nodes(&nodemask) > 1)
+		test_xapic_ipi(run_secs, delay_usecs, true);
+	else
+		__TEST_REQUIRE(!force_migrate,
+			       "Need at least 2 NUMA nodes to do migration (nodemask = 0x%lx)",
+			       nodemask);
+
 	return 0;
 }