From b2ce2307e8547e420fcce9fbad46119814f7f810 Mon Sep 17 00:00:00 2001
From: Alexey Makhalov <amakhalov@vmware.com>
Date: Tue, 9 May 2017 12:39:57 -0700
Subject: [PATCH] x86/vmware: pv-ops boot_clock
---
arch/x86/include/asm/paravirt.h | 5 +++++
arch/x86/include/asm/paravirt_types.h | 5 +++++
arch/x86/kernel/cpu/vmware.c | 24 +++++++++++++++++++++++-
arch/x86/kernel/head_64.S | 8 ++++++++
arch/x86/kernel/paravirt.c | 7 +++++++
arch/x86/kernel/setup.c | 9 +++++++++
6 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index d49bbf4..e0e5b8e 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -184,6 +184,11 @@ static inline u64 paravirt_steal_clock(int cpu)
return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
}
+static inline void paravirt_read_boot_clock64(struct timespec64 *ts)
+{
+ PVOP_VCALL1(pv_time_ops.read_boot_clock64, ts);
+}
+
static inline unsigned long long paravirt_read_pmc(int counter)
{
return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 180bc0b..b6280a9 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -53,7 +53,8 @@ struct mm_struct;
struct desc_struct;
struct task_struct;
struct cpumask;
+struct timespec64;
struct flush_tlb_info;
struct mmu_gather;
/*
@@ -99,6 +100,7 @@ struct pv_lazy_ops {
struct pv_time_ops {
unsigned long long (*sched_clock)(void);
unsigned long long (*steal_clock)(int cpu);
+ void (*read_boot_clock64)(struct timespec64 *ts);
} __no_randomize_layout;
struct pv_cpu_ops {
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index 20509ee..e299357 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -101,6 +101,7 @@ static struct cyc2ns_data vmware_cyc2ns __ro_after_init;
static int vmw_sched_clock __initdata = 1;
static DEFINE_PER_CPU_DECRYPTED(struct vmware_steal_time, steal_time) __aligned(64);
static int has_steal_clock = 0;
+uint64_t __initdata tsc_at_head;
static u64 vmware_clock_get_cycles(struct clocksource *cs)
{
@@ -123,6 +124,26 @@ struct clocksource * __init clocksource_default_clock(void)
return &clocksource_vmware;
}
+/* Function to read the exact time the system has been started. It will be
+ used as zero time for monotonic clock */
+static void vmware_read_boot_clock64(struct timespec64 *ts)
+{
+ struct timespec64 now;
+ u64 delta_nsec;
+ u32 rem;
+
+ read_persistent_clock64(&now);
+ delta_nsec = mul_u64_u32_shr(rdtsc(), vmware_cyc2ns.cyc2ns_mul,
+ vmware_cyc2ns.cyc2ns_shift);
+ delta_nsec -= vmware_cyc2ns.cyc2ns_offset;
+ ts->tv_sec = now.tv_sec - div_s64_rem(delta_nsec, NSEC_PER_SEC, &rem);
+ ts->tv_nsec = now.tv_nsec - rem;
+ while (unlikely(ts->tv_nsec < 0)) {
+ ts->tv_sec--;
+ ts->tv_nsec += NSEC_PER_SEC;
+ }
+}
+
static __init int setup_vmw_sched_clock(char *s)
{
vmw_sched_clock = 0;
@@ -143,7 +164,7 @@ static unsigned long long vmware_sched_clock(void)
static void __init vmware_cyc2ns_setup(void)
{
struct cyc2ns_data *d = &vmware_cyc2ns;
- unsigned long long tsc_now = rdtsc();
+ unsigned long long tsc_now = tsc_at_head;
clocks_calc_mult_shift(&d->cyc2ns_mul, &d->cyc2ns_shift,
vmware_tsc_khz, NSEC_PER_MSEC, 0);
@@ -347,6 +368,7 @@ static void __init vmware_paravirt_ops_setup(void)
#endif
}
clocksource_register_khz(&clocksource_vmware, vmware_tsc_khz);
+ pv_time_ops.read_boot_clock64 = vmware_read_boot_clock64;
}
#else
#define vmware_paravirt_ops_setup() do {} while (0)
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 8344dd2..60ad9ac 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -107,6 +107,14 @@ ENTRY(secondary_startup_64)
* after the boot processor executes this code.
*/
+ /*
+ * Read a TSC value first
+ */
+ rdtsc
+ shl $0x20, %rdx
+ or %rax, %rdx
+ mov %rdx, tsc_at_head(%rip)
+
/* Sanitize CPU configuration */
call verify_cpu
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 930c883..f6b3799 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -219,6 +219,12 @@ static u64 native_steal_clock(int cpu)
return 0;
}
+static void native_read_boot_clock64(struct timespec64 *ts)
+{
+ ts->tv_sec = 0;
+ ts->tv_nsec = 0;
+}
+
/* These are in entry.S */
extern void native_iret(void);
extern void native_usergs_sysret64(void);
@@ -326,6 +332,7 @@ struct pv_init_ops pv_init_ops = {
struct pv_time_ops pv_time_ops = {
.sched_clock = native_sched_clock,
.steal_clock = native_steal_clock,
+ .read_boot_clock64 = native_read_boot_clock64,
};
__visible struct pv_irq_ops pv_irq_ops = {
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 74b4472..3a6ab94 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1318,3 +1318,13 @@ static int __init register_kernel_offset_dumper(void)
return 0;
}
__initcall(register_kernel_offset_dumper);
+
+
+/* We need to define a real function for read_boot_clock64, to override the
+ weak default version */
+#ifdef CONFIG_PARAVIRT
+void read_boot_clock64(struct timespec64 *ts)
+{
+ paravirt_read_boot_clock64(ts);
+}
+#endif