From 3dad5371924736819950b964bbc28ce32e420c12 Mon Sep 17 00:00:00 2001 From: Alexey Makhalov <amakhalov@vmware.com> Date: Fri, 22 Dec 2017 15:23:06 -0800 Subject: [PATCH] x86/vmware: STA support Use callbacks via the state machine ot enable or disable STA per CPU. local_irq_disable() is used because smp_call_function_single() used to invoke the function with interrupts disabled. --- arch/x86/kernel/cpu/vmware.c | 185 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 179 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 50ed5769..fa6ea7dc 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -30,8 +30,12 @@ #include <asm/hypervisor.h> #include <asm/timer.h> #include <asm/apic.h> +#include <linux/sched.h> +#include <linux/cpu.h> #include <linux/kmsg_dump.h> #include <linux/frame.h> +#include <linux/notifier.h> +#include <linux/reboot.h> #undef pr_fmt #define pr_fmt(fmt) "vmware: " fmt @@ -48,15 +52,57 @@ #define VMWARE_PORT_CMD_VCPU_RESERVED 31 #define VMWARE_PORT_CMD_MESSAGE 30 #define VMWARE_HB_PORT_CMD_MESSAGE 0 +#define VMWARE_PORT_CMD_STEALCLOCK 91 +# define STEALCLOCK_IS_NOT_AVALIABLE -1 +# define STEALCLOCK_IS_DISABLED 0 +# define STEALCLOCK_IS_ENABLED 1 +#define VMWARE_PORT_CMD_MESSAGE 30 +#define VMWARE_HB_PORT_CMD_MESSAGE 0 #define VMWARE_PORT(cmd, eax, ebx, ecx, edx) \ + VMWARE_PORT2(cmd, eax, ebx, ecx, edx, UINT_MAX) + +#define VMWARE_PORT2(cmd, eax, ebx, ecx, edx, arg) \ __asm__("inl (%%dx)" : \ "=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) : \ "0"(VMWARE_HYPERVISOR_MAGIC), \ "1"(VMWARE_PORT_CMD_##cmd), \ - "2"(VMWARE_HYPERVISOR_PORT), "3"(UINT_MAX) : \ + "2"(VMWARE_HYPERVISOR_PORT), "3"(arg) : \ "memory"); +struct vmware_steal_time { + uint64_t clock; /* stolen time counter in units of vtsc */ + uint64_t reserved[7]; +}; +static DEFINE_PER_CPU(struct vmware_steal_time, steal_time) __aligned(64); +static int has_steal_clock = 0; + +static int vmware_cmd_stealclock(uint32_t arg1, uint32_t arg2) +{ + uint32_t result, info; + __asm__ __volatile__ ("inl (%%dx)" + : "=a" (result), + "=c" (info) + : "a" (VMWARE_HYPERVISOR_MAGIC), + "c" (VMWARE_PORT_CMD_STEALCLOCK), + "d" (VMWARE_HYPERVISOR_PORT), + "b" (0), + "S" (arg1), + "D" (arg2)); + return result; +} +#define STEALCLOCK_ENABLE(pa) \ + (vmware_cmd_stealclock((pa) >> 32, (pa) & 0xffffffff) \ + == STEALCLOCK_IS_ENABLED) + +#define STEALCLOCK_DISABLE() \ + vmware_cmd_stealclock(0, 1) + +static int vmware_is_stealclock_available(void) +{ + return STEALCLOCK_DISABLE() != STEALCLOCK_IS_NOT_AVALIABLE; +} + static unsigned long vmware_tsc_khz __ro_after_init; static inline int __vmware_platform(void) @@ -99,7 +145,7 @@ static unsigned long long vmware_sched_clock(void) return ns; } -static void __init vmware_sched_clock_setup(void) +static void __init vmware_cyc2ns_setup(void) { struct cyc2ns_data *d = &vmware_cyc2ns; unsigned long long tsc_now = rdtsc(); @@ -109,17 +155,144 @@ static void __init vmware_sched_clock_setup(void) d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul, d->cyc2ns_shift); - pv_time_ops.sched_clock = vmware_sched_clock; - pr_info("using sched offset of %llu ns\n", d->cyc2ns_offset); + pr_info("using clock offset of %llu ns\n", d->cyc2ns_offset); +} + +static uint64_t vmware_steal_clock(int cpu) +{ + struct vmware_steal_time *steal; + + steal = &per_cpu(steal_time, cpu); + return mul_u64_u32_shr(steal->clock, vmware_cyc2ns.cyc2ns_mul, + vmware_cyc2ns.cyc2ns_shift); +} + +static void vmware_register_steal_time(void) +{ + int cpu = smp_processor_id(); + struct vmware_steal_time *st = &per_cpu(steal_time, cpu); + + if (!has_steal_clock) + return; + + memset(st, 0, sizeof(*st)); + + if (!STEALCLOCK_ENABLE(slow_virt_to_phys(st))) { + has_steal_clock = 0; + return; + } + + pr_info("vmware-stealtime: cpu %d, pa %llx\n", + cpu, (unsigned long long) slow_virt_to_phys(st)); +} + +void vmware_disable_steal_time(void) +{ + if (!has_steal_clock) + return; + + STEALCLOCK_DISABLE(); +} + +static void vmware_guest_cpu_init(void) +{ + if (has_steal_clock) + vmware_register_steal_time(); +} + +static void vmware_pv_guest_cpu_reboot(void *unused) +{ + vmware_disable_steal_time(); +} + +static int vmware_pv_reboot_notify(struct notifier_block *nb, + unsigned long code, void *unused) +{ + if (code == SYS_RESTART) + on_each_cpu(vmware_pv_guest_cpu_reboot, NULL, 1); + return NOTIFY_DONE; +} + +static struct notifier_block vmware_pv_reboot_nb = { + .notifier_call = vmware_pv_reboot_notify, +}; + +#ifdef CONFIG_SMP +static void __init vmware_smp_prepare_boot_cpu(void) +{ + vmware_guest_cpu_init(); + native_smp_prepare_boot_cpu(); +} + +static int vmware_cpu_online(unsigned int cpu) +{ + local_irq_disable(); + vmware_guest_cpu_init(); + local_irq_enable(); + return 0; } +static int vmware_cpu_down_prepare(unsigned int cpu) +{ + local_irq_disable(); + vmware_disable_steal_time(); + local_irq_enable(); + return 0; +} +#endif + +static int sta_enabled __initdata = 1; /* steal time accounting */ +static __init int parse_no_vmw_sta(char *arg) +{ + sta_enabled = 0; + return 0; +} + +early_param("no-vmw-sta", parse_no_vmw_sta); + +static __init int activate_jump_labels(void) +{ + if (has_steal_clock) { + static_key_slow_inc(¶virt_steal_enabled); + if (sta_enabled) + static_key_slow_inc(¶virt_steal_rq_enabled); + } + + return 0; +} +arch_initcall(activate_jump_labels); + + static void __init vmware_paravirt_ops_setup(void) { pv_info.name = "VMware hypervisor"; pv_cpu_ops.io_delay = paravirt_nop; - if (vmware_tsc_khz && vmw_sched_clock) - vmware_sched_clock_setup(); + if (vmware_tsc_khz == 0) + return; + + vmware_cyc2ns_setup(); + + if (vmw_sched_clock) + pv_time_ops.sched_clock = vmware_sched_clock; + + if (vmware_is_stealclock_available()) { + has_steal_clock = 1; + pv_time_ops.steal_clock = vmware_steal_clock; + } + + if (has_steal_clock) { + /* We use reboot notifier only to disable steal clock */ + register_reboot_notifier(&vmware_pv_reboot_nb); +#ifdef CONFIG_SMP + smp_ops.smp_prepare_boot_cpu = vmware_smp_prepare_boot_cpu; + if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/vmware:online", + vmware_cpu_online, vmware_cpu_down_prepare) < 0) + pr_err("vmware_guest: Failed to install cpu hotplug callbacks\n"); +#else + vmware_guest_cpu_init(); +#endif + } } #else #define vmware_paravirt_ops_setup() do {} while (0) -- 2.11.0