vmx: add the support of XSAVE/XRSTOR to VMX guest

XSAVE/XRSTOR manages the existing and future processor extended states
on x86 architecture.

The XSAVE/XRSTOR infrastructure is defined in Intel SDMs:
http://www.intel.com/products/processor/manuals/

The patch uses the classical CR0.TS based algorithm to manage the
states on context switch.  At present, we know 3 bits in the
XFEATURE_ENABLED_MASK: FPU, SSE and YMM.  YMM is defined in Intel AVX
Programming Reference: http://software.intel.com/sites/avx/

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>

7 files changed, 109 insertions, 5 deletions

diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 06436361a8..07ff823c82 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -98,6 +98,7 @@
 #define X86_FEATURE_X2APIC	(4*32+21) /* Extended xAPIC */
 #define X86_FEATURE_POPCNT	(4*32+23) /* POPCNT instruction */
 #define X86_FEATURE_XSAVE	(4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
+#define X86_FEATURE_OSXSAVE	(4*32+27) /* OSXSAVE */
 #define X86_FEATURE_HYPERVISOR	(4*32+31) /* Running under some hypervisor */
 
 /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
@@ -189,6 +190,9 @@
                                  && boot_cpu_has(X86_FEATURE_FFXSR))
 
 #define cpu_has_x2apic          boot_cpu_has(X86_FEATURE_X2APIC)
+
+#define cpu_has_xsave           boot_cpu_has(X86_FEATURE_XSAVE)
+
 #define cpu_has_arch_perfmon    boot_cpu_has(X86_FEATURE_ARCH_PERFMON)
 
 #endif /* __ASM_I386_CPUFEATURE_H */
diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index 6edbe3b215..a608a87330 100644
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -440,7 +440,7 @@ unsigned long pv_guest_cr4_fixup(unsigned long guest_cr4);
       | ((v)->domain->arch.vtsc ? X86_CR4_TSD : 0))         \
       & ~X86_CR4_DE)
 #define real_cr4_to_pv_guest_cr4(c) \
-    ((c) & ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_TSD))
+    ((c) & ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_TSD | X86_CR4_OSXSAVE))
 
 void domain_cpuid(struct domain *d,
                   unsigned int  input,
diff --git a/xen/include/asm-x86/hvm/hvm.h b/xen/include/asm-x86/hvm/hvm.h
index 327a1c9737..563e211929 100644
--- a/xen/include/asm-x86/hvm/hvm.h
+++ b/xen/include/asm-x86/hvm/hvm.h
@@ -268,7 +268,8 @@ static inline int hvm_do_pmu_interrupt(struct cpu_user_regs *regs)
        (X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD |       \
         X86_CR4_DE  | X86_CR4_PSE | X86_CR4_PAE |       \
         X86_CR4_MCE | X86_CR4_PGE | X86_CR4_PCE |       \
-        X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT)))
+        X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT |           \
+        (cpu_has_xsave ? X86_CR4_OSXSAVE : 0))))
 
 /* These exceptions must always be intercepted. */
 #define HVM_TRAP_MASK ((1U << TRAP_machine_check) | (1U << TRAP_invalid_op))
diff --git a/xen/include/asm-x86/hvm/vcpu.h b/xen/include/asm-x86/hvm/vcpu.h
index cd24177232..cc97de490a 100644
--- a/xen/include/asm-x86/hvm/vcpu.h
+++ b/xen/include/asm-x86/hvm/vcpu.h
@@ -47,6 +47,15 @@ struct hvm_vcpu {
      */
     unsigned long       hw_cr[5];
 
+    /*
+     * The save area for Processor Extended States and the bitmask of the
+     * XSAVE/XRSTOR features. They are used by: 1) when a vcpu (which has
+     * dirtied FPU/SSE) is scheduled out we XSAVE the states here; 2) in
+     * #NM handler, we XRSTOR the states we XSAVE-ed;
+     */
+    void *xsave_area;
+    uint64_t xfeature_mask;
+
     struct vlapic       vlapic;
     s64                 cache_tsc_offset;
     u64                 guest_time;
diff --git a/xen/include/asm-x86/hvm/vmx/vmx.h b/xen/include/asm-x86/hvm/vmx/vmx.h
index 7bff68d145..2c248bf043 100644
--- a/xen/include/asm-x86/hvm/vmx/vmx.h
+++ b/xen/include/asm-x86/hvm/vmx/vmx.h
@@ -111,6 +111,7 @@ void vmx_update_debug_state(struct vcpu *v);
 #define EXIT_REASON_EPT_VIOLATION       48
 #define EXIT_REASON_EPT_MISCONFIG       49
 #define EXIT_REASON_WBINVD              54
+#define EXIT_REASON_XSETBV              55
 
 /*
  * Interruption-information format
diff --git a/xen/include/asm-x86/i387.h b/xen/include/asm-x86/i387.h
index b0ce77aac3..39e0e7df56 100644
--- a/xen/include/asm-x86/i387.h
+++ b/xen/include/asm-x86/i387.h
@@ -14,6 +14,82 @@
 #include <xen/sched.h>
 #include <asm/processor.h>
 
+extern unsigned int xsave_cntxt_size;
+extern u32 xfeature_low, xfeature_high;
+
+extern void xsave_init(void);
+extern void xsave_init_save_area(void *save_area);
+
+#define XSTATE_FP       (1 << 0)
+#define XSTATE_SSE      (1 << 1)
+#define XSTATE_YMM      (1 << 2)
+#define XSTATE_FP_SSE   (XSTATE_FP | XSTATE_SSE)
+#define XCNTXT_MASK     (XSTATE_FP | XSTATE_SSE | XSTATE_YMM)
+#define XSTATE_YMM_OFFSET  (512 + 64)
+#define XSTATE_YMM_SIZE    256
+
+struct xsave_struct
+{
+    struct { char x[512]; } fpu_sse;         /* FPU/MMX, SSE */
+
+    struct {
+        u64 xstate_bv;
+        u64 reserved[7];
+    } xsave_hdr;                            /* The 64-byte header */
+
+    struct { char x[XSTATE_YMM_SIZE]; } ymm; /* YMM */
+    char   data[];                           /* Future new states */
+} __attribute__ ((packed, aligned (64)));
+
+#define XCR_XFEATURE_ENABLED_MASK   0
+
+#ifdef CONFIG_X86_64
+#define REX_PREFIX "0x48, "
+#else
+#define REX_PREFIX
+#endif
+
+static inline void xsetbv(u32 index, u64 xfeature_mask)
+{
+    u32 hi = xfeature_mask >> 32;
+    u32 lo = (u32)xfeature_mask;
+
+    asm volatile (".byte 0x0f,0x01,0xd1" :: "c" (index),
+            "a" (lo), "d" (hi));
+}
+
+static inline void set_xcr0(u64 xfeature_mask)
+{
+    xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeature_mask);
+}
+
+static inline void xsave(struct vcpu *v)
+{
+    u64 mask = v->arch.hvm_vcpu.xfeature_mask | XSTATE_FP_SSE;
+    u32 lo = mask, hi = mask >> 32;
+    struct xsave_struct *ptr;
+
+    ptr =(struct xsave_struct *)v->arch.hvm_vcpu.xsave_area;
+
+    asm volatile (".byte " REX_PREFIX "0x0f,0xae,0x27"
+        :
+        : "a" (lo), "d" (hi), "D"(ptr)
+        : "memory");
+}
+
+static inline void xrstor(struct vcpu *v)
+{
+    u64 mask = v->arch.hvm_vcpu.xfeature_mask | XSTATE_FP_SSE;
+    u32 lo = mask, hi = mask >> 32;
+    struct xsave_struct *ptr;
+
+    ptr =(struct xsave_struct *)v->arch.hvm_vcpu.xsave_area;
+
+    asm volatile (".byte " REX_PREFIX "0x0f,0xae,0x2f"
+        :
+        : "m" (*ptr), "a" (lo), "d" (hi), "D"(ptr));
+}
+
 extern void init_fpu(void);
 extern void save_init_fpu(struct vcpu *v);
 extern void restore_fpu(struct vcpu *v);
@@ -36,10 +112,22 @@ static inline void setup_fpu(struct vcpu *v)
     if ( !v->fpu_dirtied )
     {
         v->fpu_dirtied = 1;
-        if ( v->fpu_initialised )
-            restore_fpu(v);
+        if ( cpu_has_xsave && is_hvm_vcpu(v) )
+        {
+            if ( !v->fpu_initialised )
+                v->fpu_initialised = 1;
+
+            set_xcr0(v->arch.hvm_vcpu.xfeature_mask | XSTATE_FP_SSE);
+            xrstor(v);
+            set_xcr0(v->arch.hvm_vcpu.xfeature_mask);
+        }
         else
-            init_fpu();
+        {
+            if ( v->fpu_initialised )
+                restore_fpu(v);
+            else
+                init_fpu();
+        }
     }
 }
 
diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h
index 61c9dde637..7b09adecd6 100644
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -83,6 +83,7 @@
 #define X86_CR4_OSXMMEXCPT	0x0400	/* enable unmasked SSE exceptions */
 #define X86_CR4_VMXE		0x2000  /* enable VMX */
 #define X86_CR4_SMXE		0x4000  /* enable SMX */
+#define X86_CR4_OSXSAVE	0x40000 /* enable XSAVE/XRSTOR */
 
 /*
  * Trap/fault mnemonics.