x86, hvm: Lots of MTRR/PAT emulation cleanup.

 - Move MTRR MSR initialisation into hvmloader.
 - Simplify initialisation logic by overlaying UC on default WB rather
   than vice versa.
 - Clean up hypervisor HVM MTRR/PAE code's interface with rest of
   hypervisor.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

6 files changed, 112 insertions, 9 deletions

diff --git a/tools/firmware/hvmloader/Makefile b/tools/firmware/hvmloader/Makefile
index ae1fdc869d..826338560e 100644
--- a/tools/firmware/hvmloader/Makefile
+++ b/tools/firmware/hvmloader/Makefile
@@ -28,8 +28,9 @@ LOADADDR = 0x100000
 
 CFLAGS += $(CFLAGS_include) -I.
 
-SRCS = hvmloader.c mp_tables.c util.c smbios.c 32bitbios_support.c smp.c
-OBJS = $(patsubst %.c,%.o,$(SRCS))
+SRCS  = hvmloader.c mp_tables.c util.c smbios.c 
+SRCS += 32bitbios_support.c smp.c cacheattr.c
+OBJS  = $(patsubst %.c,%.o,$(SRCS))
 
 .PHONY: all
 all: hvmloader
diff --git a/tools/firmware/hvmloader/acpi/build.c b/tools/firmware/hvmloader/acpi/build.c
index 5043f5de2a..252a055728 100644
--- a/tools/firmware/hvmloader/acpi/build.c
+++ b/tools/firmware/hvmloader/acpi/build.c
@@ -84,8 +84,8 @@ static int construct_bios_info_table(uint8_t *buf)
 
     bios_info->hpet_present = hpet_exists(ACPI_HPET_ADDRESS);
 
-    bios_info->pci_min = 0xf0000000;
-    bios_info->pci_len = 0x0c000000;
+    bios_info->pci_min = PCI_MEMBASE;
+    bios_info->pci_len = PCI_MEMSIZE;
 
     return align16(sizeof(*bios_info));
 }
diff --git a/tools/firmware/hvmloader/cacheattr.c b/tools/firmware/hvmloader/cacheattr.c
new file mode 100644
index 0000000000..5dc2c7f807
--- /dev/null
+++ b/tools/firmware/hvmloader/cacheattr.c
@@ -0,0 +1,99 @@
+/*
+ * cacheattr.c: MTRR and PAT initialisation.
+ *
+ * Copyright (c) 2008, Citrix Systems, Inc.
+ * 
+ * Authors:
+ *    Keir Fraser <keir.fraser@citrix.com>
+ * 
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include "util.h"
+#include "config.h"
+
+#define MSR_MTRRphysBase(reg) (0x200 + 2 * (reg))
+#define MSR_MTRRphysMask(reg) (0x200 + 2 * (reg) + 1)
+#define MSR_MTRRcap          0x00fe
+#define MSR_MTRRfix64K_00000 0x0250
+#define MSR_MTRRfix16K_80000 0x0258
+#define MSR_MTRRfix16K_A0000 0x0259
+#define MSR_MTRRfix4K_C0000  0x0268
+#define MSR_MTRRfix4K_C8000  0x0269
+#define MSR_MTRRfix4K_D0000  0x026a
+#define MSR_MTRRfix4K_D8000  0x026b
+#define MSR_MTRRfix4K_E0000  0x026c
+#define MSR_MTRRfix4K_E8000  0x026d
+#define MSR_MTRRfix4K_F0000  0x026e
+#define MSR_MTRRfix4K_F8000  0x026f
+#define MSR_PAT              0x0277
+#define MSR_MTRRdefType      0x02ff
+
+void cacheattr_init(void)
+{
+    uint32_t eax, ebx, ecx, edx;
+    uint64_t mtrr_cap, mtrr_def, content, addr_mask;
+    unsigned int i, nr_var_ranges, phys_bits = 36;
+
+    /* Does the CPU support architectural MTRRs? */
+    cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
+    if ( !(edx & (1u << 12)) )
+         return;
+
+    /* Find the physical address size for this CPU. */
+    cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
+    if ( eax >= 0x80000008 )
+    {
+        cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
+        phys_bits = (uint8_t)eax;
+    }
+
+    printf("%u-bit phys ... ", phys_bits);
+
+    addr_mask = ((1ull << phys_bits) - 1) & ~((1ull << 12) - 1);
+    mtrr_cap = rdmsr(MSR_MTRRcap);
+    mtrr_def = (1u << 11) | 6; /* E, default type WB */
+
+    /* Fixed-range MTRRs supported? */
+    if ( mtrr_cap & (1u << 8) )
+    {
+        /* 0x00000-0x9ffff: Write Back (WB) */
+        content = 0x0606060606060606ull;
+        wrmsr(MSR_MTRRfix64K_00000, content);
+        wrmsr(MSR_MTRRfix16K_80000, content);
+        /* 0xa0000-0xbffff: Write Combining (WC) */
+        if ( mtrr_cap & (1u << 10) ) /* WC supported? */
+            content = 0x0101010101010101ull;
+        wrmsr(MSR_MTRRfix16K_A0000, content);
+        /* 0xc0000-0xfffff: Write Back (WB) */
+        content = 0x0606060606060606ull;
+        for ( i = 0; i < 8; i++ )
+            wrmsr(MSR_MTRRfix4K_C0000 + i, content);
+        mtrr_def |= 1u << 10; /* FE */
+        printf("fixed MTRRs ... ");
+    }
+
+    /* Variable-range MTRRs supported? */
+    nr_var_ranges = (uint8_t)mtrr_cap;
+    if ( nr_var_ranges != 0 )
+    {
+        /* A single UC range covering PCI space. */
+        wrmsr(MSR_MTRRphysBase(0), PCI_MEMBASE);
+        wrmsr(MSR_MTRRphysMask(0),
+              ((uint64_t)(int32_t)PCI_MEMBASE & addr_mask) | (1u << 11));
+        printf("var MTRRs ... ");
+    }
+
+    wrmsr(MSR_MTRRdefType, mtrr_def);
+}
diff --git a/tools/firmware/hvmloader/config.h b/tools/firmware/hvmloader/config.h
index 24d3e47159..cde2056f80 100644
--- a/tools/firmware/hvmloader/config.h
+++ b/tools/firmware/hvmloader/config.h
@@ -11,6 +11,9 @@
 #define PCI_ISA_DEVFN       0x08    /* dev 1, fn 0 */
 #define PCI_ISA_IRQ_MASK    0x0c20U /* ISA IRQs 5,10,11 are PCI connected */
 
+#define PCI_MEMBASE         0xf0000000
+#define PCI_MEMSIZE         0x0c000000
+
 #define ROMBIOS_SEG            0xF000
 #define ROMBIOS_BEGIN          0x000F0000
 #define ROMBIOS_SIZE           0x00010000
diff --git a/tools/firmware/hvmloader/hvmloader.c b/tools/firmware/hvmloader/hvmloader.c
index 31a39b787f..b3e075fd8a 100644
--- a/tools/firmware/hvmloader/hvmloader.c
+++ b/tools/firmware/hvmloader/hvmloader.c
@@ -159,7 +159,7 @@ static void pci_setup(void)
     struct resource {
         uint32_t base, max;
     } *resource;
-    struct resource mem_resource = { 0xf0000000, 0xfc000000 };
+    struct resource mem_resource = { PCI_MEMBASE, PCI_MEMBASE + PCI_MEMSIZE };
     struct resource io_resource  = { 0xc000, 0x10000 };
 
     /* Create a list of device BARs in descending order of size. */
diff --git a/tools/firmware/hvmloader/smp.c b/tools/firmware/hvmloader/smp.c
index 62934dc274..4d88b5b760 100644
--- a/tools/firmware/hvmloader/smp.c
+++ b/tools/firmware/hvmloader/smp.c
@@ -69,10 +69,12 @@ asm (
     "    .text                       \n"
     );
 
+extern void cacheattr_init(void);
+
 /*static*/ void ap_start(void)
 {
     printf(" - CPU%d ... ", ap_cpuid);
-
+    cacheattr_init();
     printf("done.\n");
     wmb();
     ap_callin = 1;
@@ -122,12 +124,10 @@ void smp_initialise(void)
 {
     unsigned int i, nr_cpus = get_vcpu_nr();
 
-    if ( nr_cpus <= 1 )
-        return;
-
     memcpy((void *)AP_BOOT_EIP, ap_boot_start, ap_boot_end - ap_boot_start);
 
     printf("Multiprocessor initialisation:\n");
+    ap_start();
     for ( i = 1; i < nr_cpus; i++ )
         boot_cpu(i);
 }