| fc081194 |
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -30,6 +30,8 @@
#include <asm/hypervisor.h>
#include <asm/timer.h>
#include <asm/apic.h>
+#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/kmsg_dump.h>
#undef pr_fmt
@@ -47,15 +49,57 @@
#define VMWARE_PORT_CMD_VCPU_RESERVED 31
#define VMWARE_PORT_CMD_MESSAGE 30
#define VMWARE_HB_PORT_CMD_MESSAGE 0
+#define VMWARE_PORT_CMD_STEALCLOCK 91
+# define STEALCLOCK_IS_NOT_AVALIABLE -1
+# define STEALCLOCK_IS_DISABLED 0
+# define STEALCLOCK_IS_ENABLED 1
+#define VMWARE_PORT_CMD_MESSAGE 30
+#define VMWARE_HB_PORT_CMD_MESSAGE 0
#define VMWARE_PORT(cmd, eax, ebx, ecx, edx) \
+ VMWARE_PORT2(cmd, eax, ebx, ecx, edx, UINT_MAX)
+
+#define VMWARE_PORT2(cmd, eax, ebx, ecx, edx, arg) \
__asm__("inl (%%dx)" : \
"=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) : \
"0"(VMWARE_HYPERVISOR_MAGIC), \
"1"(VMWARE_PORT_CMD_##cmd), \
- "2"(VMWARE_HYPERVISOR_PORT), "3"(UINT_MAX) : \
+ "2"(VMWARE_HYPERVISOR_PORT), "3"(arg) : \
"memory");
+struct vmware_steal_time {
+ uint64_t clock; /* stolen time counter in units of vtsc */
+ uint64_t reserved[7];
+};
+static DEFINE_PER_CPU(struct vmware_steal_time, steal_time) __aligned(64);
+static int has_steal_clock = 0;
+
+static int vmware_cmd_stealclock(uint32_t arg1, uint32_t arg2)
+{
+ uint32_t result, info;
+ __asm__ __volatile__ ("inl (%%dx)"
+ : "=a" (result),
+ "=c" (info)
+ : "a" (VMWARE_HYPERVISOR_MAGIC),
+ "c" (VMWARE_PORT_CMD_STEALCLOCK),
+ "d" (VMWARE_HYPERVISOR_PORT),
+ "b" (0),
+ "S" (arg1),
+ "D" (arg2));
+ return result;
+}
+#define STEALCLOCK_ENABLE(pa) \
+ (vmware_cmd_stealclock((pa) >> 32, (pa) & 0xffffffff) \
+ == STEALCLOCK_IS_ENABLED)
+
+#define STEALCLOCK_DISABLE() \
+ vmware_cmd_stealclock(0, 1)
+
+static int vmware_is_stealclock_available(void)
+{
+ return STEALCLOCK_DISABLE() != STEALCLOCK_IS_NOT_AVALIABLE;
+}
+
static unsigned long vmware_tsc_khz __ro_after_init;
static inline int __vmware_platform(void)
@@ -98,7 +142,7 @@ static unsigned long long vmware_sched_clock(void)
return ns;
}
-static void __init vmware_sched_clock_setup(void)
+static void __init vmware_cyc2ns_setup(void)
{
struct cyc2ns_data *d = &vmware_cyc2ns;
unsigned long long tsc_now = rdtsc();
@@ -108,17 +152,144 @@ static void __init vmware_sched_clock_setup(void)
d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul,
d->cyc2ns_shift);
- pv_time_ops.sched_clock = vmware_sched_clock;
- pr_info("using sched offset of %llu ns\n", d->cyc2ns_offset);
+ pr_info("using clock offset of %llu ns\n", d->cyc2ns_offset);
}
+static uint64_t vmware_steal_clock(int cpu)
+{
+ struct vmware_steal_time *steal;
+
+ steal = &per_cpu(steal_time, cpu);
+ return mul_u64_u32_shr(steal->clock, vmware_cyc2ns.cyc2ns_mul,
+ vmware_cyc2ns.cyc2ns_shift);
+}
+
+static void vmware_register_steal_time(void)
+{
+ int cpu = smp_processor_id();
+ struct vmware_steal_time *st = &per_cpu(steal_time, cpu);
+
+ if (!has_steal_clock)
+ return;
+
+ memset(st, 0, sizeof(*st));
+
+ if (!STEALCLOCK_ENABLE(slow_virt_to_phys(st))) {
+ has_steal_clock = 0;
+ return;
+ }
+
+ pr_info("vmware-stealtime: cpu %d, pa %llx\n",
+ cpu, (unsigned long long) slow_virt_to_phys(st));
+}
+
+void vmware_disable_steal_time(void)
+{
+ if (!has_steal_clock)
+ return;
+
+ STEALCLOCK_DISABLE();
+}
+
+static void vmware_guest_cpu_init(void)
+{
+ if (has_steal_clock)
+ vmware_register_steal_time();
+}
+
+#ifdef CONFIG_SMP
+static void __init vmware_smp_prepare_boot_cpu(void)
+{
+ vmware_guest_cpu_init();
+ native_smp_prepare_boot_cpu();
+}
+
+static void vmware_guest_cpu_online(void *dummy)
+{
+ vmware_guest_cpu_init();
+}
+
+static void vmware_guest_cpu_offline(void *dummy)
+{
+ vmware_disable_steal_time();
+}
+
+static int vmware_cpu_notify(struct notifier_block *self, unsigned long action,
+ void *hcpu)
+{
+ int cpu = (unsigned long)hcpu;
+ switch (action) {
+ case CPU_ONLINE:
+ case CPU_DOWN_FAILED:
+ case CPU_ONLINE_FROZEN:
+ smp_call_function_single(cpu, vmware_guest_cpu_online,
+ NULL, 0);
+ break;
+ case CPU_DOWN_PREPARE:
+ case CPU_DOWN_PREPARE_FROZEN:
+ smp_call_function_single(cpu, vmware_guest_cpu_offline,
+ NULL, 1);
+ break;
+ default:
+ break;
+ }
+ return NOTIFY_OK;
+}
+
+static struct notifier_block vmware_cpu_notifier = {
+ .notifier_call = vmware_cpu_notify,
+};
+#endif
+
+static int sta_enabled __initdata = 1; /* steal time accounting */
+static __init int parse_no_vmw_sta(char *arg)
+{
+ sta_enabled = 0;
+ return 0;
+}
+
+early_param("no-vmw-sta", parse_no_vmw_sta);
+
+static __init int activate_jump_labels(void)
+{
+ if (has_steal_clock) {
+ static_key_slow_inc(¶virt_steal_enabled);
+ if (sta_enabled)
+ static_key_slow_inc(¶virt_steal_rq_enabled);
+ }
+
+ return 0;
+}
+arch_initcall(activate_jump_labels);
+
+
static void __init vmware_paravirt_ops_setup(void)
{
pv_info.name = "VMware hypervisor";
pv_cpu_ops.io_delay = paravirt_nop;
- if (vmware_tsc_khz && vmw_sched_clock)
- vmware_sched_clock_setup();
+ if (vmware_tsc_khz == 0)
+ return;
+
+ vmware_cyc2ns_setup();
+
+ if (vmw_sched_clock)
+ pv_time_ops.sched_clock = vmware_sched_clock;
+
+ if (vmware_is_stealclock_available()) {
+ has_steal_clock = 1;
+ pv_time_ops.steal_clock = vmware_steal_clock;
+ }
+
+ /* vmware_cpu_notifier is used only by STA */
+ if (has_steal_clock) {
+#ifdef CONFIG_SMP
+ smp_ops.smp_prepare_boot_cpu = vmware_smp_prepare_boot_cpu;
+ register_cpu_notifier(&vmware_cpu_notifier);
+#else
+ vmware_guest_cpu_init();
+#endif
+ }
}
#else
#define vmware_paravirt_ops_setup() do {} while (0)
@@ -163,7 +334,6 @@ static void __init vmware_platform_setup(void)
#endif
vmware_paravirt_ops_setup();
-
kmsg_dump_register(&kmsg_dumper);
}
|