From 3dad5371924736819950b964bbc28ce32e420c12 Mon Sep 17 00:00:00 2001
From: Alexey Makhalov <amakhalov@vmware.com>
Date: Fri, 22 Dec 2017 15:23:06 -0800
Subject: [PATCH] x86/vmware: STA support
Use callbacks via the state machine ot enable or disable STA per CPU.
local_irq_disable() is used because smp_call_function_single() used to invoke
the function with interrupts disabled.
---
arch/x86/kernel/cpu/vmware.c | 185 +++++++++++++++++++++++++++++++++++++++++--
1 file changed, 179 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index 50ed5769..fa6ea7dc 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -30,8 +30,12 @@
#include <asm/hypervisor.h>
#include <asm/timer.h>
#include <asm/apic.h>
+#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/kmsg_dump.h>
#include <linux/frame.h>
+#include <linux/notifier.h>
+#include <linux/reboot.h>
#undef pr_fmt
#define pr_fmt(fmt) "vmware: " fmt
@@ -48,15 +52,57 @@
#define VMWARE_PORT_CMD_VCPU_RESERVED 31
#define VMWARE_PORT_CMD_MESSAGE 30
#define VMWARE_HB_PORT_CMD_MESSAGE 0
+#define VMWARE_PORT_CMD_STEALCLOCK 91
+# define STEALCLOCK_IS_NOT_AVALIABLE -1
+# define STEALCLOCK_IS_DISABLED 0
+# define STEALCLOCK_IS_ENABLED 1
+#define VMWARE_PORT_CMD_MESSAGE 30
+#define VMWARE_HB_PORT_CMD_MESSAGE 0
#define VMWARE_PORT(cmd, eax, ebx, ecx, edx) \
+ VMWARE_PORT2(cmd, eax, ebx, ecx, edx, UINT_MAX)
+
+#define VMWARE_PORT2(cmd, eax, ebx, ecx, edx, arg) \
__asm__("inl (%%dx)" : \
"=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) : \
"0"(VMWARE_HYPERVISOR_MAGIC), \
"1"(VMWARE_PORT_CMD_##cmd), \
- "2"(VMWARE_HYPERVISOR_PORT), "3"(UINT_MAX) : \
+ "2"(VMWARE_HYPERVISOR_PORT), "3"(arg) : \
"memory");
+struct vmware_steal_time {
+ uint64_t clock; /* stolen time counter in units of vtsc */
+ uint64_t reserved[7];
+};
+static DEFINE_PER_CPU(struct vmware_steal_time, steal_time) __aligned(64);
+static int has_steal_clock = 0;
+
+static int vmware_cmd_stealclock(uint32_t arg1, uint32_t arg2)
+{
+ uint32_t result, info;
+ __asm__ __volatile__ ("inl (%%dx)"
+ : "=a" (result),
+ "=c" (info)
+ : "a" (VMWARE_HYPERVISOR_MAGIC),
+ "c" (VMWARE_PORT_CMD_STEALCLOCK),
+ "d" (VMWARE_HYPERVISOR_PORT),
+ "b" (0),
+ "S" (arg1),
+ "D" (arg2));
+ return result;
+}
+#define STEALCLOCK_ENABLE(pa) \
+ (vmware_cmd_stealclock((pa) >> 32, (pa) & 0xffffffff) \
+ == STEALCLOCK_IS_ENABLED)
+
+#define STEALCLOCK_DISABLE() \
+ vmware_cmd_stealclock(0, 1)
+
+static int vmware_is_stealclock_available(void)
+{
+ return STEALCLOCK_DISABLE() != STEALCLOCK_IS_NOT_AVALIABLE;
+}
+
static unsigned long vmware_tsc_khz __ro_after_init;
static inline int __vmware_platform(void)
@@ -99,7 +145,7 @@ static unsigned long long vmware_sched_clock(void)
return ns;
}
-static void __init vmware_sched_clock_setup(void)
+static void __init vmware_cyc2ns_setup(void)
{
struct cyc2ns_data *d = &vmware_cyc2ns;
unsigned long long tsc_now = rdtsc();
@@ -109,17 +155,144 @@ static void __init vmware_sched_clock_setup(void)
d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul,
d->cyc2ns_shift);
- pv_time_ops.sched_clock = vmware_sched_clock;
- pr_info("using sched offset of %llu ns\n", d->cyc2ns_offset);
+ pr_info("using clock offset of %llu ns\n", d->cyc2ns_offset);
+}
+
+static uint64_t vmware_steal_clock(int cpu)
+{
+ struct vmware_steal_time *steal;
+
+ steal = &per_cpu(steal_time, cpu);
+ return mul_u64_u32_shr(steal->clock, vmware_cyc2ns.cyc2ns_mul,
+ vmware_cyc2ns.cyc2ns_shift);
+}
+
+static void vmware_register_steal_time(void)
+{
+ int cpu = smp_processor_id();
+ struct vmware_steal_time *st = &per_cpu(steal_time, cpu);
+
+ if (!has_steal_clock)
+ return;
+
+ memset(st, 0, sizeof(*st));
+
+ if (!STEALCLOCK_ENABLE(slow_virt_to_phys(st))) {
+ has_steal_clock = 0;
+ return;
+ }
+
+ pr_info("vmware-stealtime: cpu %d, pa %llx\n",
+ cpu, (unsigned long long) slow_virt_to_phys(st));
+}
+
+void vmware_disable_steal_time(void)
+{
+ if (!has_steal_clock)
+ return;
+
+ STEALCLOCK_DISABLE();
+}
+
+static void vmware_guest_cpu_init(void)
+{
+ if (has_steal_clock)
+ vmware_register_steal_time();
+}
+
+static void vmware_pv_guest_cpu_reboot(void *unused)
+{
+ vmware_disable_steal_time();
+}
+
+static int vmware_pv_reboot_notify(struct notifier_block *nb,
+ unsigned long code, void *unused)
+{
+ if (code == SYS_RESTART)
+ on_each_cpu(vmware_pv_guest_cpu_reboot, NULL, 1);
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block vmware_pv_reboot_nb = {
+ .notifier_call = vmware_pv_reboot_notify,
+};
+
+#ifdef CONFIG_SMP
+static void __init vmware_smp_prepare_boot_cpu(void)
+{
+ vmware_guest_cpu_init();
+ native_smp_prepare_boot_cpu();
+}
+
+static int vmware_cpu_online(unsigned int cpu)
+{
+ local_irq_disable();
+ vmware_guest_cpu_init();
+ local_irq_enable();
+ return 0;
}
+static int vmware_cpu_down_prepare(unsigned int cpu)
+{
+ local_irq_disable();
+ vmware_disable_steal_time();
+ local_irq_enable();
+ return 0;
+}
+#endif
+
+static int sta_enabled __initdata = 1; /* steal time accounting */
+static __init int parse_no_vmw_sta(char *arg)
+{
+ sta_enabled = 0;
+ return 0;
+}
+
+early_param("no-vmw-sta", parse_no_vmw_sta);
+
+static __init int activate_jump_labels(void)
+{
+ if (has_steal_clock) {
+ static_key_slow_inc(¶virt_steal_enabled);
+ if (sta_enabled)
+ static_key_slow_inc(¶virt_steal_rq_enabled);
+ }
+
+ return 0;
+}
+arch_initcall(activate_jump_labels);
+
+
static void __init vmware_paravirt_ops_setup(void)
{
pv_info.name = "VMware hypervisor";
pv_cpu_ops.io_delay = paravirt_nop;
- if (vmware_tsc_khz && vmw_sched_clock)
- vmware_sched_clock_setup();
+ if (vmware_tsc_khz == 0)
+ return;
+
+ vmware_cyc2ns_setup();
+
+ if (vmw_sched_clock)
+ pv_time_ops.sched_clock = vmware_sched_clock;
+
+ if (vmware_is_stealclock_available()) {
+ has_steal_clock = 1;
+ pv_time_ops.steal_clock = vmware_steal_clock;
+ }
+
+ if (has_steal_clock) {
+ /* We use reboot notifier only to disable steal clock */
+ register_reboot_notifier(&vmware_pv_reboot_nb);
+#ifdef CONFIG_SMP
+ smp_ops.smp_prepare_boot_cpu = vmware_smp_prepare_boot_cpu;
+ if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/vmware:online",
+ vmware_cpu_online, vmware_cpu_down_prepare) < 0)
+ pr_err("vmware_guest: Failed to install cpu hotplug callbacks\n");
+#else
+ vmware_guest_cpu_init();
+#endif
+ }
}
#else
#define vmware_paravirt_ops_setup() do {} while (0)
--
2.11.0